dataloader/src/data_loader.py

import sys

running_which_env='prd'


from os.path import dirname, abspath, join
ROOT_DIR = abspath(join(dirname(abspath(__file__)), ".."))
from loguru import logger
logger.remove()
logger.add(sys.stderr, level="INFO")
logger.add(ROOT_DIR+"/logs/{time:YYYYMMDD-HHmmss}"+f"_{running_which_env}.log", rotation="10 MB", compression="zip", level="INFO")



import pandas as pd

from DDBfm import DDBfm
from TSLfm import TSLfm

from code_list import code_list_pickel

def run_add_1day_code_init_minKline(date,code_list):
    ddb = DDBfm(running_which_env)

    code_list_filtered = []
    for code in code_list:
        if ddb.search_code_date_in_tb(ddb.ddf_hft_mink_tbname,date,code):
            logger.warning(f"Possible duplicates on {date} and {code}")
        else:
            code_list_filtered.append(code)
    if len(code_list_filtered)==0:
        return 0

    with TSLfm() as tsl:
        df = tsl.process_result_data_type(tsl.get_mkt_min_k(date,date,code_list_filtered))
    if not df.empty:
        logger.info(f'Getting a df of {df.shape}: {code_list[0][:-4]} on {date}')
        ddb.append_hft_table(ddb.ddf_hft_mink_tbname,df)
    
def run_create_db_minKline():
    date = '20221101'
    with TSLfm() as tsl:
        code_list=['T2212']
        df = tsl.process_result_data_type(tsl.get_mkt_min_k(date,date,code_list))
        # print(df)
    ddb = DDBfm(running_which_env)
    ddb.create_hft_database()
    ddb.create_hft_table(ddb.ddf_hft_mink_tbname,df)
    
def run():

    all_code_dict_by_init={}
    for c in code_list_pickel:
        init = c[:-4]
        if init in all_code_dict_by_init:
            all_code_dict_by_init[init].append(c)
        else:
            all_code_dict_by_init[init]=[c]

    # print(all_code_dict_by_init)

    start_date='2022-09-30'
    end_date='2022-11-09'
    allDates = pd.date_range(start_date, end_date, freq ='D')
    allDates = [i.replace('-','') for i in list(allDates.astype('str'))]

    for date in allDates:
        for code_init in all_code_dict_by_init:
            code_list = all_code_dict_by_init[code_init]

            run_add_1day_code_init_minKline(date,code_list)


    # date = '20221101'
    # with TSLfm() as tsl:
    #     # code_list = tsl.get_code_list("国债期货")
    #     # code_list += tsl.get_code_list("股指期货")
    #     # code_list += tsl.get_code_list("上市期货")
    #     # code_list=sorted(list(set(code_list)))

    #     # print(code_list_pickel)
    #     code_list=['CF2211']
    #     df = tsl.process_result_data_type(tsl.get_mkt_min_k(date,date,code_list))
    #     print(df)

    # ddb = DDBfm('prd')
    # ddb.create_hft_database()
    # ddb.create_hft_table(ddb.ddf_hft_mink_tbname,df)
    
    # if ddb.search_code_date_in_tb(ddb.ddf_hft_mink_tbname,date,'CF2211'):
        # logger.warning(f"Possible duplicates on {date} and ")

    # ddb.append_hft_table(ddb.ddf_hft_mink_tbname,df)

if __name__ == '__main__':
    run()
    # run_create_db_minKline()