Python数据处理

第一章 怎么把一列String全部转化为datetime

---

---

百度了一下很多都是抄的，用datetime.datetime.strptime(str(a), “%Y%m%d”)，是按照格式取出年月日的数据然后填进模板

from datetime import datetime, date, time
 
In [27]: dt = datetime(2011, 10, 29, 20, 30, 21)
 
In [28]: dt.day
Out[28]: 29
 
In [29]: dt.minute
Out[29]: 30
 
In [30]: dt.date()
Out[30]: datetime.date(2011, 10, 29)
 
In [31]: dt.time()
Out[31]: datetime.time(20, 30, 21)
 
In [32]: dt.strftime("%m/%d/%Y %H:%M")
Out[32]: '10/29/2011 20:30'
 
In [33]: datetime.strptime("20181006","%Y%m%d")
Out[33]: datetime.datetime(2018, 10, 6, 0, 0)

# -*- coding:utf-8 -*-
import datetime
 
 
# 计算两个日期之间相差多少年, 多少天 , 多少月;
 
# begin_date = input("请输入开始日期:")
# end_date = input("请输入结束日期:")
begin = '2017/01/01'
end = '2018/12/31'
# print(type(begin_date), begin_date)  # <class 'str'> 2017/01/01
 
# 将 字符串开始日期 转成 日期格式   注意 : 格式一定要对应 /
begin_date = datetime.datetime.strptime(begin, '%Y/%m/%d')
print("开始:", begin_date)
end_date = datetime.datetime.strptime(end, '%Y/%m/%d')
print("结束:", end_date)
 
# 从开始到结束之间有多少个月
month = (end_date.year - begin_date.year)*12 + (end_date.month - begin_date.month)
print(month)

他是把单个的字符串转换为datetime,但因为会扫描所有的datetime的格式来进行匹配所以会有性能问题可以自己按规定的原则来重构这个函数

调用pandas.to_datetime

数据源是由数据库或者离线csv文件读取而来，会形成dataframe格式,在pandas包里边会有to_datetime()方法来进行列格式的转换,但是他要求被转换的数据列为String格式，具体应用可以参见该方法的源码

应用场景如下（）：

import pandas as pd
     # 编码格式非UTF-8,需要自己调整到UTF-8,或者在read_scv里加入encoding属性调整编码格式
  online_retail=pd.read_csv('F:\customer_k_means\online_retail_1.csv')
     # string格式转datetime格式
  online_retail['InvoiceDate'] = pd.to_datetime(online_retail['InvoiceDate'])

源码

代码如下（示例）：

@overload
def to_datetime(
    arg: Union[List, Tuple],
    errors: str = ...,
    dayfirst: bool = ...,
    yearfirst: bool = ...,
    utc: Optional[bool] = ...,
    format: Optional[str] = ...,
    exact: bool = ...,
    unit: Optional[str] = ...,
    infer_datetime_format: bool = ...,
    origin=...,
    cache: bool = ...,
) -> DatetimeIndex:
    ...


def to_datetime(
    arg: DatetimeScalarOrArrayConvertible,
    errors: str = "raise",
    dayfirst: bool = False,
    yearfirst: bool = False,
    utc: Optional[bool] = None,
    format: Optional[str] = None,
    exact: bool = True,
    unit: Optional[str] = None,
    infer_datetime_format: bool = False,
    origin="unix",
    cache: bool = True,
) -> Union[DatetimeIndex, "Series", DatetimeScalar, "NaTType"]:
    """
    Convert argument to datetime.

    Parameters
    ----------
    arg : int, float, str, datetime, list, tuple, 1-d array, Series, DataFrame/dict-like
        The object to convert to a datetime.
    errors : {'ignore', 'raise', 'coerce'}, default 'raise'
        - If 'raise', then invalid parsing will raise an exception.
        - If 'coerce', then invalid parsing will be set as NaT.
        - If 'ignore', then invalid parsing will return the input.
    dayfirst : bool, default False
        Specify a date parse order if `arg` is str or its list-likes.
        If True, parses dates with the day first, eg 10/11/12 is parsed as
        2012-11-10.
        Warning: dayfirst=True is not strict, but will prefer to parse
        with day first (this is a known bug, based on dateutil behavior).
    yearfirst : bool, default False
        Specify a date parse order if `arg` is str or its list-likes.

        - If True parses dates with the year first, eg 10/11/12 is parsed as
          2010-11-12.
        - If both dayfirst and yearfirst are True, yearfirst is preceded (same
          as dateutil).

        Warning: yearfirst=True is not strict, but will prefer to parse
        with year first (this is a known bug, based on dateutil behavior).
    utc : bool, default None
        Return UTC DatetimeIndex if True (converting any tz-aware
        datetime.datetime objects as well).
    format : str, default None
        The strftime to parse time, eg "%d/%m/%Y", note that "%f" will parse
        all the way up to nanoseconds.
        See strftime documentation for more information on choices:
        https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior.
    exact : bool, True by default
        Behaves as:
        - If True, require an exact format match.
        - If False, allow the format to match anywhere in the target string.

    unit : str, default 'ns'
        The unit of the arg (D,s,ms,us,ns) denote the unit, which is an
        integer or float number. This will be based off the origin.
        Example, with unit='ms' and origin='unix' (the default), this
        would calculate the number of milliseconds to the unix epoch start.
    infer_datetime_format : bool, default False
        If True and no `format` is given, attempt to infer the format of the
        datetime strings based on the first non-NaN element,
        and if it can be inferred, switch to a faster method of parsing them.
        In some cases this can increase the parsing speed by ~5-10x.
    origin : scalar, default 'unix'
        Define the reference date. The numeric values would be parsed as number
        of units (defined by `unit`) since this reference date.

        - If 'unix' (or POSIX) time; origin is set to 1970-01-01.
        - If 'julian', unit must be 'D', and origin is set to beginning of
          Julian Calendar. Julian day number 0 is assigned to the day starting
          at noon on January 1, 4713 BC.
        - If Timestamp convertible, origin is set to Timestamp identified by
          origin.
    cache : bool, default True
        If True, use a cache of unique, converted dates to apply the datetime
        conversion. May produce significant speed-up when parsing duplicate
        date strings, especially ones with timezone offsets. The cache is only
        used when there are at least 50 values. The presence of out-of-bounds
        values will render the cache unusable and may slow down parsing.

        .. versionadded:: 0.23.0

        .. versionchanged:: 0.25.0
            - changed default value from False to True.

    Returns
    -------
    datetime
        If parsing succeeded.
        Return type depends on input:

        - list-like: DatetimeIndex
        - Series: Series of datetime64 dtype
        - scalar: Timestamp

        In case when it is not possible to return designated types (e.g. when
        any element of input is before Timestamp.min or after Timestamp.max)
        return will have datetime.datetime type (or corresponding
        array/Series).

    See Also
    --------
    DataFrame.astype : Cast argument to a specified dtype.
    to_timedelta : Convert argument to timedelta.
    convert_dtypes : Convert dtypes.

    Examples
    --------
    Assembling a datetime from multiple columns of a DataFrame. The keys can be
    common abbreviations like ['year', 'month', 'day', 'minute', 'second',
    'ms', 'us', 'ns']) or plurals of the same

    >>> df = pd.DataFrame({'year': [2015, 2016],
    ...                    'month': [2, 3],
    ...                    'day': [4, 5]})
    >>> pd.to_datetime(df)
    0   2015-02-04
    1   2016-03-05
    dtype: datetime64[ns]

    If a date does not meet the `timestamp limitations
    <https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html
    #timeseries-timestamp-limits>`_, passing errors='ignore'
    will return the original input instead of raising any exception.

    Passing errors='coerce' will force an out-of-bounds date to NaT,
    in addition to forcing non-dates (or non-parseable dates) to NaT.

    >>> pd.to_datetime('13000101', format='%Y%m%d', errors='ignore')
    datetime.datetime(1300, 1, 1, 0, 0)
    >>> pd.to_datetime('13000101', format='%Y%m%d', errors='coerce')
    NaT

    Passing infer_datetime_format=True can often-times speedup a parsing
    if its not an ISO8601 format exactly, but in a regular format.

    >>> s = pd.Series(['3/11/2000', '3/12/2000', '3/13/2000'] * 1000)
    >>> s.head()
    0    3/11/2000
    1    3/12/2000
    2    3/13/2000
    3    3/11/2000
    4    3/12/2000
    dtype: object

    >>> %timeit pd.to_datetime(s, infer_datetime_format=True)  # doctest: +SKIP
    100 loops, best of 3: 10.4 ms per loop

    >>> %timeit pd.to_datetime(s, infer_datetime_format=False)  # doctest: +SKIP
    1 loop, best of 3: 471 ms per loop

    Using a unix epoch time

    >>> pd.to_datetime(1490195805, unit='s')
    Timestamp('2017-03-22 15:16:45')
    >>> pd.to_datetime(1490195805433502912, unit='ns')
    Timestamp('2017-03-22 15:16:45.433502912')

    .. warning:: For float arg, precision rounding might happen. To prevent
        unexpected behavior use a fixed-width exact type.

    Using a non-unix epoch origin

    >>> pd.to_datetime([1, 2, 3], unit='D',
    ...                origin=pd.Timestamp('1960-01-01'))
    DatetimeIndex(['1960-01-02', '1960-01-03', '1960-01-04'], \
dtype='datetime64[ns]', freq=None)
    """
    if arg is None:
        return None

    if origin != "unix":
        arg = _adjust_to_origin(arg, origin, unit)

    tz = "utc" if utc else None
    convert_listlike = partial(
        _convert_listlike_datetimes,
        tz=tz,
        unit=unit,
        dayfirst=dayfirst,
        yearfirst=yearfirst,
        errors=errors,
        exact=exact,
        infer_datetime_format=infer_datetime_format,
    )

    if isinstance(arg, Timestamp):
        result = arg
        if tz is not None:
            if arg.tz is not None:
                result = result.tz_convert(tz)
            else:
                result = result.tz_localize(tz)
    elif isinstance(arg, ABCSeries):
        cache_array = _maybe_cache(arg, format, cache, convert_listlike)
        if not cache_array.empty:
            result = arg.map(cache_array)
        else:
            values = convert_listlike(arg._values, format)
            result = arg._constructor(values, index=arg.index, name=arg.name)
    elif isinstance(arg, (ABCDataFrame, abc.MutableMapping)):
        result = _assemble_from_unit_mappings(arg, errors, tz)
    elif isinstance(arg, Index):
        cache_array = _maybe_cache(arg, format, cache, convert_listlike)
        if not cache_array.empty:
            result = _convert_and_box_cache(arg, cache_array, name=arg.name)
        else:
            result = convert_listlike(arg, format, name=arg.name)
    elif is_list_like(arg):
        try:
            cache_array = _maybe_cache(arg, format, cache, convert_listlike)
        except tslibs.OutOfBoundsDatetime:
            # caching attempts to create a DatetimeIndex, which may raise
            # an OOB. If that's the desired behavior, then just reraise...
            if errors == "raise":
                raise
            # ... otherwise, continue without the cache.
            from pandas import Series

            cache_array = Series([], dtype=object)  # just an empty array
        if not cache_array.empty:
            result = _convert_and_box_cache(arg, cache_array)
        else:
            result = convert_listlike(arg, format)
    else:
        result = convert_listlike(np.array([arg]), format)[0]

    return result


# mappings for assembling units
_unit_map = {
    "year": "year",
    "years": "year",
    "month": "month",
    "months": "month",
    "day": "day",
    "days": "day",
    "hour": "h",
    "hours": "h",
    "minute": "m",
    "minutes": "m",
    "second": "s",
    "seconds": "s",
    "ms": "ms",
    "millisecond": "ms",
    "milliseconds": "ms",
    "us": "us",
    "microsecond": "us",
    "microseconds": "us",
    "ns": "ns",
    "nanosecond": "ns",
    "nanoseconds": "ns",
}

---

总结

除了会有pandas现成的类型转换的方法to_datetime也有其他的转换方法，和SQL的to_char,to_date,substr之类的也会存在
之后会写SQL和pandas函数的对应机制