针对pulse-transit的工具

2025-02-22 16:12:02 +08:00
commit 6bc25b4e3a
7719 changed files with 1530886 additions and 0 deletions
--- a/dist/client/pandas/tests/io/parser/test_na_values.py
+++ b/dist/client/pandas/tests/io/parser/test_na_values.py
@@ -0,0 +1,656 @@
+"""
+Tests that NA values are properly handled during
+parsing for all of the parsers defined in parsers.py
+"""
+from io import StringIO
+
+import numpy as np
+import pytest
+
+from pandas._libs.parsers import STR_NA_VALUES
+
+from pandas import (
+    DataFrame,
+    Index,
+    MultiIndex,
+)
+import pandas._testing as tm
+
+skip_pyarrow = pytest.mark.usefixtures("pyarrow_skip")
+xfail_pyarrow = pytest.mark.usefixtures("pyarrow_xfail")
+
+
+@skip_pyarrow
+def test_string_nas(all_parsers):
+    parser = all_parsers
+    data = """A,B,C
+a,b,c
+d,,f
+,g,h
+"""
+    result = parser.read_csv(StringIO(data))
+    expected = DataFrame(
+        [["a", "b", "c"], ["d", np.nan, "f"], [np.nan, "g", "h"]],
+        columns=["A", "B", "C"],
+    )
+    tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow
+def test_detect_string_na(all_parsers):
+    parser = all_parsers
+    data = """A,B
+foo,bar
+NA,baz
+NaN,nan
+"""
+    expected = DataFrame(
+        [["foo", "bar"], [np.nan, "baz"], [np.nan, np.nan]], columns=["A", "B"]
+    )
+    result = parser.read_csv(StringIO(data))
+    tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow
+@pytest.mark.parametrize(
+    "na_values",
+    [
+        ["-999.0", "-999"],
+        [-999, -999.0],
+        [-999.0, -999],
+        ["-999.0"],
+        ["-999"],
+        [-999.0],
+        [-999],
+    ],
+)
+@pytest.mark.parametrize(
+    "data",
+    [
+        """A,B
+-999,1.2
+2,-999
+3,4.5
+""",
+        """A,B
+-999,1.200
+2,-999.000
+3,4.500
+""",
+    ],
+)
+def test_non_string_na_values(all_parsers, data, na_values):
+    # see gh-3611: with an odd float format, we can't match
+    # the string "999.0" exactly but still need float matching
+    parser = all_parsers
+    expected = DataFrame([[np.nan, 1.2], [2.0, np.nan], [3.0, 4.5]], columns=["A", "B"])
+
+    result = parser.read_csv(StringIO(data), na_values=na_values)
+    tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow
+def test_default_na_values(all_parsers):
+    _NA_VALUES = {
+        "-1.#IND",
+        "1.#QNAN",
+        "1.#IND",
+        "-1.#QNAN",
+        "#N/A",
+        "N/A",
+        "n/a",
+        "NA",
+        "<NA>",
+        "#NA",
+        "NULL",
+        "null",
+        "NaN",
+        "nan",
+        "-NaN",
+        "-nan",
+        "#N/A N/A",
+        "",
+    }
+    assert _NA_VALUES == STR_NA_VALUES
+
+    parser = all_parsers
+    nv = len(_NA_VALUES)
+
+    def f(i, v):
+        if i == 0:
+            buf = ""
+        elif i > 0:
+            buf = "".join([","] * i)
+
+        buf = f"{buf}{v}"
+
+        if i < nv - 1:
+            joined = "".join([","] * (nv - i - 1))
+            buf = f"{buf}{joined}"
+
+        return buf
+
+    data = StringIO("\n".join([f(i, v) for i, v in enumerate(_NA_VALUES)]))
+    expected = DataFrame(np.nan, columns=range(nv), index=range(nv))
+
+    result = parser.read_csv(data, header=None)
+    tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow
+@pytest.mark.parametrize("na_values", ["baz", ["baz"]])
+def test_custom_na_values(all_parsers, na_values):
+    parser = all_parsers
+    data = """A,B,C
+ignore,this,row
+1,NA,3
+-1.#IND,5,baz
+7,8,NaN
+"""
+    expected = DataFrame(
+        [[1.0, np.nan, 3], [np.nan, 5, np.nan], [7, 8, np.nan]], columns=["A", "B", "C"]
+    )
+    result = parser.read_csv(StringIO(data), na_values=na_values, skiprows=[1])
+    tm.assert_frame_equal(result, expected)
+
+
+def test_bool_na_values(all_parsers):
+    data = """A,B,C
+True,False,True
+NA,True,False
+False,NA,True"""
+    parser = all_parsers
+    result = parser.read_csv(StringIO(data))
+    expected = DataFrame(
+        {
+            "A": np.array([True, np.nan, False], dtype=object),
+            "B": np.array([False, True, np.nan], dtype=object),
+            "C": [True, False, True],
+        }
+    )
+    tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow
+def test_na_value_dict(all_parsers):
+    data = """A,B,C
+foo,bar,NA
+bar,foo,foo
+foo,bar,NA
+bar,foo,foo"""
+    parser = all_parsers
+    df = parser.read_csv(StringIO(data), na_values={"A": ["foo"], "B": ["bar"]})
+    expected = DataFrame(
+        {
+            "A": [np.nan, "bar", np.nan, "bar"],
+            "B": [np.nan, "foo", np.nan, "foo"],
+            "C": [np.nan, "foo", np.nan, "foo"],
+        }
+    )
+    tm.assert_frame_equal(df, expected)
+
+
+@skip_pyarrow
+@pytest.mark.parametrize(
+    "index_col,expected",
+    [
+        (
+            [0],
+            DataFrame({"b": [np.nan], "c": [1], "d": [5]}, index=Index([0], name="a")),
+        ),
+        (
+            [0, 2],
+            DataFrame(
+                {"b": [np.nan], "d": [5]},
+                index=MultiIndex.from_tuples([(0, 1)], names=["a", "c"]),
+            ),
+        ),
+        (
+            ["a", "c"],
+            DataFrame(
+                {"b": [np.nan], "d": [5]},
+                index=MultiIndex.from_tuples([(0, 1)], names=["a", "c"]),
+            ),
+        ),
+    ],
+)
+def test_na_value_dict_multi_index(all_parsers, index_col, expected):
+    data = """\
+a,b,c,d
+0,NA,1,5
+"""
+    parser = all_parsers
+    result = parser.read_csv(StringIO(data), na_values=set(), index_col=index_col)
+    tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow
+@pytest.mark.parametrize(
+    "kwargs,expected",
+    [
+        (
+            {},
+            DataFrame(
+                {
+                    "A": ["a", "b", np.nan, "d", "e", np.nan, "g"],
+                    "B": [1, 2, 3, 4, 5, 6, 7],
+                    "C": ["one", "two", "three", np.nan, "five", np.nan, "seven"],
+                }
+            ),
+        ),
+        (
+            {"na_values": {"A": [], "C": []}, "keep_default_na": False},
+            DataFrame(
+                {
+                    "A": ["a", "b", "", "d", "e", "nan", "g"],
+                    "B": [1, 2, 3, 4, 5, 6, 7],
+                    "C": ["one", "two", "three", "nan", "five", "", "seven"],
+                }
+            ),
+        ),
+        (
+            {"na_values": ["a"], "keep_default_na": False},
+            DataFrame(
+                {
+                    "A": [np.nan, "b", "", "d", "e", "nan", "g"],
+                    "B": [1, 2, 3, 4, 5, 6, 7],
+                    "C": ["one", "two", "three", "nan", "five", "", "seven"],
+                }
+            ),
+        ),
+        (
+            {"na_values": {"A": [], "C": []}},
+            DataFrame(
+                {
+                    "A": ["a", "b", np.nan, "d", "e", np.nan, "g"],
+                    "B": [1, 2, 3, 4, 5, 6, 7],
+                    "C": ["one", "two", "three", np.nan, "five", np.nan, "seven"],
+                }
+            ),
+        ),
+    ],
+)
+def test_na_values_keep_default(all_parsers, kwargs, expected):
+    data = """\
+A,B,C
+a,1,one
+b,2,two
+,3,three
+d,4,nan
+e,5,five
+nan,6,
+g,7,seven
+"""
+    parser = all_parsers
+    result = parser.read_csv(StringIO(data), **kwargs)
+    tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow
+def test_no_na_values_no_keep_default(all_parsers):
+    # see gh-4318: passing na_values=None and
+    # keep_default_na=False yields 'None" as a na_value
+    data = """\
+A,B,C
+a,1,None
+b,2,two
+,3,None
+d,4,nan
+e,5,five
+nan,6,
+g,7,seven
+"""
+    parser = all_parsers
+    result = parser.read_csv(StringIO(data), keep_default_na=False)
+
+    expected = DataFrame(
+        {
+            "A": ["a", "b", "", "d", "e", "nan", "g"],
+            "B": [1, 2, 3, 4, 5, 6, 7],
+            "C": ["None", "two", "None", "nan", "five", "", "seven"],
+        }
+    )
+    tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow
+def test_no_keep_default_na_dict_na_values(all_parsers):
+    # see gh-19227
+    data = "a,b\n,2"
+    parser = all_parsers
+    result = parser.read_csv(
+        StringIO(data), na_values={"b": ["2"]}, keep_default_na=False
+    )
+    expected = DataFrame({"a": [""], "b": [np.nan]})
+    tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow
+def test_no_keep_default_na_dict_na_scalar_values(all_parsers):
+    # see gh-19227
+    #
+    # Scalar values shouldn't cause the parsing to crash or fail.
+    data = "a,b\n1,2"
+    parser = all_parsers
+    df = parser.read_csv(StringIO(data), na_values={"b": 2}, keep_default_na=False)
+    expected = DataFrame({"a": [1], "b": [np.nan]})
+    tm.assert_frame_equal(df, expected)
+
+
+@skip_pyarrow
+@pytest.mark.parametrize("col_zero_na_values", [113125, "113125"])
+def test_no_keep_default_na_dict_na_values_diff_reprs(all_parsers, col_zero_na_values):
+    # see gh-19227
+    data = """\
+113125,"blah","/blaha",kjsdkj,412.166,225.874,214.008
+729639,"qwer","",asdfkj,466.681,,252.373
+"""
+    parser = all_parsers
+    expected = DataFrame(
+        {
+            0: [np.nan, 729639.0],
+            1: [np.nan, "qwer"],
+            2: ["/blaha", np.nan],
+            3: ["kjsdkj", "asdfkj"],
+            4: [412.166, 466.681],
+            5: ["225.874", ""],
+            6: [np.nan, 252.373],
+        }
+    )
+
+    result = parser.read_csv(
+        StringIO(data),
+        header=None,
+        keep_default_na=False,
+        na_values={2: "", 6: "214.008", 1: "blah", 0: col_zero_na_values},
+    )
+    tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow
+@pytest.mark.parametrize(
+    "na_filter,row_data",
+    [
+        (True, [[1, "A"], [np.nan, np.nan], [3, "C"]]),
+        (False, [["1", "A"], ["nan", "B"], ["3", "C"]]),
+    ],
+)
+def test_na_values_na_filter_override(all_parsers, na_filter, row_data):
+    data = """\
+A,B
+1,A
+nan,B
+3,C
+"""
+    parser = all_parsers
+    result = parser.read_csv(StringIO(data), na_values=["B"], na_filter=na_filter)
+
+    expected = DataFrame(row_data, columns=["A", "B"])
+    tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow
+def test_na_trailing_columns(all_parsers):
+    parser = all_parsers
+    data = """Date,Currency,Symbol,Type,Units,UnitPrice,Cost,Tax
+2012-03-14,USD,AAPL,BUY,1000
+2012-05-12,USD,SBUX,SELL,500"""
+
+    # Trailing columns should be all NaN.
+    result = parser.read_csv(StringIO(data))
+    expected = DataFrame(
+        [
+            ["2012-03-14", "USD", "AAPL", "BUY", 1000, np.nan, np.nan, np.nan],
+            ["2012-05-12", "USD", "SBUX", "SELL", 500, np.nan, np.nan, np.nan],
+        ],
+        columns=[
+            "Date",
+            "Currency",
+            "Symbol",
+            "Type",
+            "Units",
+            "UnitPrice",
+            "Cost",
+            "Tax",
+        ],
+    )
+    tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow
+@pytest.mark.parametrize(
+    "na_values,row_data",
+    [
+        (1, [[np.nan, 2.0], [2.0, np.nan]]),
+        ({"a": 2, "b": 1}, [[1.0, 2.0], [np.nan, np.nan]]),
+    ],
+)
+def test_na_values_scalar(all_parsers, na_values, row_data):
+    # see gh-12224
+    parser = all_parsers
+    names = ["a", "b"]
+    data = "1,2\n2,1"
+
+    result = parser.read_csv(StringIO(data), names=names, na_values=na_values)
+    expected = DataFrame(row_data, columns=names)
+    tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow
+def test_na_values_dict_aliasing(all_parsers):
+    parser = all_parsers
+    na_values = {"a": 2, "b": 1}
+    na_values_copy = na_values.copy()
+
+    names = ["a", "b"]
+    data = "1,2\n2,1"
+
+    expected = DataFrame([[1.0, 2.0], [np.nan, np.nan]], columns=names)
+    result = parser.read_csv(StringIO(data), names=names, na_values=na_values)
+
+    tm.assert_frame_equal(result, expected)
+    tm.assert_dict_equal(na_values, na_values_copy)
+
+
+@skip_pyarrow
+def test_na_values_dict_col_index(all_parsers):
+    # see gh-14203
+    data = "a\nfoo\n1"
+    parser = all_parsers
+    na_values = {0: "foo"}
+
+    result = parser.read_csv(StringIO(data), na_values=na_values)
+    expected = DataFrame({"a": [np.nan, 1]})
+    tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow
+@pytest.mark.parametrize(
+    "data,kwargs,expected",
+    [
+        (
+            str(2**63) + "\n" + str(2**63 + 1),
+            {"na_values": [2**63]},
+            DataFrame([str(2**63), str(2**63 + 1)]),
+        ),
+        (str(2**63) + ",1" + "\n,2", {}, DataFrame([[str(2**63), 1], ["", 2]])),
+        (str(2**63) + "\n1", {"na_values": [2**63]}, DataFrame([np.nan, 1])),
+    ],
+)
+def test_na_values_uint64(all_parsers, data, kwargs, expected):
+    # see gh-14983
+    parser = all_parsers
+    result = parser.read_csv(StringIO(data), header=None, **kwargs)
+    tm.assert_frame_equal(result, expected)
+
+
+def test_empty_na_values_no_default_with_index(all_parsers):
+    # see gh-15835
+    data = "a,1\nb,2"
+    parser = all_parsers
+    expected = DataFrame({"1": [2]}, index=Index(["b"], name="a"))
+
+    result = parser.read_csv(StringIO(data), index_col=0, keep_default_na=False)
+    tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow
+@pytest.mark.parametrize(
+    "na_filter,index_data", [(False, ["", "5"]), (True, [np.nan, 5.0])]
+)
+def test_no_na_filter_on_index(all_parsers, na_filter, index_data):
+    # see gh-5239
+    #
+    # Don't parse NA-values in index unless na_filter=True
+    parser = all_parsers
+    data = "a,b,c\n1,,3\n4,5,6"
+
+    expected = DataFrame({"a": [1, 4], "c": [3, 6]}, index=Index(index_data, name="b"))
+    result = parser.read_csv(StringIO(data), index_col=[1], na_filter=na_filter)
+    tm.assert_frame_equal(result, expected)
+
+
+def test_inf_na_values_with_int_index(all_parsers):
+    # see gh-17128
+    parser = all_parsers
+    data = "idx,col1,col2\n1,3,4\n2,inf,-inf"
+
+    # Don't fail with OverflowError with inf's and integer index column.
+    out = parser.read_csv(StringIO(data), index_col=[0], na_values=["inf", "-inf"])
+    expected = DataFrame(
+        {"col1": [3, np.nan], "col2": [4, np.nan]}, index=Index([1, 2], name="idx")
+    )
+    tm.assert_frame_equal(out, expected)
+
+
+@skip_pyarrow
+@pytest.mark.parametrize("na_filter", [True, False])
+def test_na_values_with_dtype_str_and_na_filter(all_parsers, na_filter):
+    # see gh-20377
+    parser = all_parsers
+    data = "a,b,c\n1,,3\n4,5,6"
+
+    # na_filter=True --> missing value becomes NaN.
+    # na_filter=False --> missing value remains empty string.
+    empty = np.nan if na_filter else ""
+    expected = DataFrame({"a": ["1", "4"], "b": [empty, "5"], "c": ["3", "6"]})
+
+    result = parser.read_csv(StringIO(data), na_filter=na_filter, dtype=str)
+    tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow
+@pytest.mark.parametrize(
+    "data, na_values",
+    [
+        ("false,1\n,1\ntrue", None),
+        ("false,1\nnull,1\ntrue", None),
+        ("false,1\nnan,1\ntrue", None),
+        ("false,1\nfoo,1\ntrue", "foo"),
+        ("false,1\nfoo,1\ntrue", ["foo"]),
+        ("false,1\nfoo,1\ntrue", {"a": "foo"}),
+    ],
+)
+def test_cast_NA_to_bool_raises_error(all_parsers, data, na_values):
+    parser = all_parsers
+    msg = (
+        "(Bool column has NA values in column [0a])|"
+        "(cannot safely convert passed user dtype of "
+        "bool for object dtyped data in column 0)"
+    )
+    with pytest.raises(ValueError, match=msg):
+        parser.read_csv(
+            StringIO(data),
+            header=None,
+            names=["a", "b"],
+            dtype={"a": "bool"},
+            na_values=na_values,
+        )
+
+
+@skip_pyarrow
+def test_str_nan_dropped(all_parsers):
+    # see gh-21131
+    parser = all_parsers
+
+    data = """File: small.csv,,
+10010010233,0123,654
+foo,,bar
+01001000155,4530,898"""
+
+    result = parser.read_csv(
+        StringIO(data),
+        header=None,
+        names=["col1", "col2", "col3"],
+        dtype={"col1": str, "col2": str, "col3": str},
+    ).dropna()
+
+    expected = DataFrame(
+        {
+            "col1": ["10010010233", "01001000155"],
+            "col2": ["0123", "4530"],
+            "col3": ["654", "898"],
+        },
+        index=[1, 3],
+    )
+
+    tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow
+def test_nan_multi_index(all_parsers):
+    # GH 42446
+    parser = all_parsers
+    data = "A,B,B\nX,Y,Z\n1,2,inf"
+
+    result = parser.read_csv(
+        StringIO(data), header=list(range(2)), na_values={("B", "Z"): "inf"}
+    )
+
+    expected = DataFrame(
+        {
+            ("A", "X"): [1],
+            ("B", "Y"): [2],
+            ("B", "Z"): [np.nan],
+        }
+    )
+
+    tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow
+def test_bool_and_nan_to_bool(all_parsers):
+    # GH#42808
+    parser = all_parsers
+    data = """0
+NaN
+True
+False
+"""
+    with pytest.raises(ValueError, match="NA values"):
+        parser.read_csv(StringIO(data), dtype="bool")
+
+
+def test_bool_and_nan_to_int(all_parsers):
+    # GH#42808
+    parser = all_parsers
+    data = """0
+NaN
+True
+False
+"""
+    with pytest.raises(ValueError, match="convert|NoneType"):
+        parser.read_csv(StringIO(data), dtype="int")
+
+
+def test_bool_and_nan_to_float(all_parsers):
+    # GH#42808
+    parser = all_parsers
+    data = """0
+NaN
+True
+False
+"""
+    result = parser.read_csv(StringIO(data), dtype="float")
+    expected = DataFrame.from_dict({"0": [np.nan, 1.0, 0.0]})
+    tm.assert_frame_equal(result, expected)