dolphin-dev/src/loader/DDBLoader.py

import importlib
import pickle
import functools
import abc

from pprint import pprint
from pathlib import Path

import numpy as np
import pandas as pd

import dolphindb as ddb
import dolphindb.settings as keys

import sqlalchemy as sa

from DDBBase import DDBBase


class DDBLoader(DDBBase):
    """
    - 放了几个公用的配置字段，包括：
        1. SQL-Server的链接参数
        2. DolphinDB的链接参数

    - 放了几个@abstractmethod在里面，不过如果不需要使用多态特性，那应该用处不大：
        1. create_ddb_database
        2. create_ddb_partition_table
    """

    mssql_config = {
        'host' : '192.168.1.7',
        'username' : 'sa',
        'password' : 'passw0rd!'
    }

    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        self.mssql_engine = sa.create_engine(
            "mssql+pyodbc://{username}:{password}@{host}/master?driver=ODBC+Driver+18+for+SQL+Server".format(**self.mssql_config),
            connect_args = {
                "TrustServerCertificate": "yes"
            }, echo=False
        )


    @abc.abstractmethod
    def create_ddb_database(self, *args, **kwargs):
        """
        创建database函数，需要被子类具体实现。
        """
        return


    @abc.abstractmethod
    def create_ddb_partition_table(self, *args, **kwargs):
        """
        创建分区表函数，需要被子类具体实现。
        """
        return 


    @staticmethod
    def tscode_to_windcode(series):
        return series.apply(lambda x : x[2:] + '.' + x[:2])


    @staticmethod
    def make_symbol(series):
        return series.astype('int32').astype('str')\
            .apply(str.zfill, args=(6,))\
            .apply(lambda code : \
                code + '.SH' if code[0] == '6' \
                else code + '.SZ')


    @staticmethod
    def make_date(series):
        # 特别是对于分红表，如果某些关键日期还未公布，则会填充0，导致日期解析失败
        series.loc[series == 0] = np.nan
        return pd.to_datetime(
            series.astype(str), format='%Y%m%d')


    @staticmethod
    def make_nparray(series):
        return series.apply(lambda x : np.array(x))


    @staticmethod
    def make_time(series):
        s_hr = series // 10000000 * 3600000
        s_min = series % 10000000 // 100000 * 60000
        s_sec = series % 100000 // 1000
        s_ms = series % 1000
        return pd.to_timedelta(s_hr + s_min + s_sec + s_ms, unit='ms')


    def make_calendar_df(self):
        print('Will create calendar dataframe from SQL Server')
        # 从KLine表查询，主要是因为KLine表最小
        with self.mssql_engine.connect() as conn:
            stat = "select distinct S_INFO_WINDCODE, TRADE_DT from Level2BytesKline.dbo.KLine"
            rs = conn.execute(stat)
            stock_date_list = [(stock_name, date) for stock_name, date in rs.fetchall()]

        df_calendar = pd.DataFrame(stock_date_list, columns=['code', 'm_nDate'])
        df_calendar['m_nDate'] = self.make_date(df_calendar['m_nDate'])
        print('Did make the DataFrame for calendar')
        return df_calendar