Merge pull request #143 from fasiondog/feature/block

Feature/block 支持 mysql/sqlite 板块信息；优化权息信息加载速度

hikyuu/data/common.py

@@ -46,16 +46,16 @@ class MARKETID:
 
 
 class STOCKTYPE:
-    BLOCK = 0  #板块
-    A = 1  #A股
-    INDEX = 2  #指数
-    B = 3  #B股
-    FUND = 4  #基金（非ETF）
-    ETF = 5  #ETF
-    ND = 6  #国债
-    BOND = 7  #其他债券
-    GEM = 8  #创业板
-    START = 9  #科创板
+    BLOCK = 0  # 板块
+    A = 1  # A股
+    INDEX = 2  # 指数
+    B = 3  # B股
+    FUND = 4  # 基金（非ETF）
+    ETF = 5  # ETF
+    ND = 6  # 国债
+    BOND = 7  # 其他债券
+    GEM = 8  # 创业板
+    START = 9  # 科创板
 
 
 def get_stktype_list(quotations=None):
@@ -127,10 +127,15 @@ def get_index_code_name_list() -> list:
     """
     获取所有股票指数代码名称列表
     从新浪获取，多次频繁调用会被封禁IP，需10分钟后再试
-    
+
     :return: [{'market_code': 'SHxxx'}, ...]
     """
-    df = ak.stock_zh_index_spot()
+    if hasattr(ak, 'stock_zh_index_spot_em'):
+        df = ak.stock_zh_index_spot_em()
+    elif hasattr(ak, 'stock_zh_index_spot_sina'):
+        df = ak.stock_zh_index_spot_sina()
+    else:
+        df = ak.stock_zh_index_spot()
     return [{'market_code': df.loc[i]['代码'].upper(), 'name': df.loc[i]['名称']} for i in range(len(df))]
 
 
@@ -174,4 +179,4 @@ def get_new_holidays():
     res.encoding = res.apparent_encoding
     ret = re.findall(r'<textarea id="data" style="display:none;">([\s\w\d\W]+)</textarea>', res.text, re.M)[0].strip()
     day = [d.split('|')[:4] for d in ret.split('\n')]
-    return [v[0] for v in day if v[2] == '中国']
+    return [v[0] for v in day if v[2] == '中国']

hikyuu/data/em_block_to_mysql.py

@@ -0,0 +1,84 @@
+#!/usr/bin/python
+# -*- coding: utf8 -*-
+#
+# Create on: 20240102
+#    Author: fasiondog
+
+import json
+import akshare  # 这里必须再导入 akshare 下，否则get_all_zsbk_info会报错
+from hikyuu.data.common import MARKET, get_stk_code_name_list
+from hikyuu.util import *
+from hikyuu.fetcher.stock.zh_block_em import *
+
+
+def em_import_block_to_mysql(connect, code_market_dict, categorys=('行业板块', '概念板块', '地域板块', '指数板块')):
+    all_block_info = {}
+
+    if '行业板块' in categorys:
+        hku_info("获取行业板块信息")
+        x = get_all_hybk_info(code_market_dict)
+        if x:
+            all_block_info["行业板块"] = x
+
+    if '行业板块' in categorys:
+        hku_info("获取概念板块信息")
+        x = get_all_gnbk_info(code_market_dict)
+        if x:
+            all_block_info["概念板块"] = x
+
+    if '行业板块' in categorys:
+        hku_info("获取地域板块信息")
+        x = get_all_dybk_info(code_market_dict)
+        if x:
+            all_block_info["地域板块"] = x
+
+    if '行业板块' in categorys:
+        hku_info("获取指数板块信息")
+        x = get_all_zsbk_info(code_market_dict)
+        if x:
+            all_block_info["指数板块"] = x
+
+    hku_info("更新数据库")
+    cur = connect.cursor()
+    sql = "delete from hku_base.block where category in ('行业板块', '概念板块', '地域板块', '指数板块')"
+    cur.execute(sql)
+
+    insert_records = []
+
+    for category in all_block_info:
+        for name in all_block_info[category]:
+            for code in all_block_info[category][name]:
+                insert_records.append((category, name, code))
+
+    if insert_records:
+        sql = "insert into hku_base.block (category, name, market_code) values (%s,%s,%s)"
+        cur.executemany(sql, insert_records)
+
+    connect.commit()
+    cur.close()
+    pass
+
+
+if __name__ == "__main__":
+    import mysql
+    from hikyuu.data.common_mysql import create_database
+
+    host = '127.0.0.1'
+    port = 3306
+    usr = 'root'
+    pwd = ''
+
+    src_dir = "D:\\TdxW_HuaTai"
+    quotations = ['stock', 'fund']  # 通达信盘后数据没有债券
+
+    connect = mysql.connector.connect(user=usr, password=pwd, host=host, port=port)
+    create_database(connect)
+
+    x = get_stk_code_name_list(MARKET.SH)
+    code_market_dict = {}
+    for v in x:
+        code_market_dict[v["code"]] = MARKET.SH
+    # print(code_market_dict)
+
+    em_import_block_to_mysql(connect, code_market_dict)
+    connect.close()

hikyuu/data/em_block_to_sqlite.py

@@ -0,0 +1,80 @@
+#!/usr/bin/python
+# -*- coding: utf8 -*-
+#
+# Create on: 20240102
+#    Author: fasiondog
+
+import json
+import akshare  # 这里必须再导入 akshare 下，否则get_all_zsbk_info会报错
+from hikyuu.data.common import MARKET, get_stk_code_name_list
+from hikyuu.util import *
+from hikyuu.fetcher.stock.zh_block_em import *
+
+
+def em_import_block_to_sqlite(connect, code_market_dict, categorys=('行业板块', '概念板块', '地域板块', '指数板块')):
+    all_block_info = {}
+
+    if '行业板块' in categorys:
+        hku_info("获取行业板块信息")
+        x = get_all_hybk_info(code_market_dict)
+        if x:
+            all_block_info["行业板块"] = x
+
+    if '概念板块' in categorys:
+        hku_info("获取概念板块信息")
+        x = get_all_gnbk_info(code_market_dict)
+        if x:
+            all_block_info["概念板块"] = x
+
+    if '地域板块' in categorys:
+        hku_info("获取地域板块信息")
+        x = get_all_dybk_info(code_market_dict)
+        if x:
+            all_block_info["地域板块"] = x
+
+    if '指数板块' in categorys:
+        hku_info("获取指数板块信息")
+        x = get_all_zsbk_info(code_market_dict)
+        if x:
+            all_block_info["指数板块"] = x
+
+    hku_info("更新数据库")
+    cur = connect.cursor()
+    sql = "delete from block where category in ('行业板块', '概念板块', '地域板块', '指数板块')"
+    cur.execute(sql)
+
+    insert_records = []
+
+    for category in all_block_info:
+        for name in all_block_info[category]:
+            for code in all_block_info[category][name]:
+                insert_records.append((category, name, code))
+
+    if insert_records:
+        sql = "insert into block (category, name, market_code) values (?,?,?)"
+        cur.executemany(sql, insert_records)
+
+    connect.commit()
+    cur.close()
+    pass
+
+
+if __name__ == "__main__":
+    import sqlite3
+    from hikyuu.data.common_sqlite3 import create_database
+
+    dest_dir = "/home/fasiondog/stock"
+    # dest_dir = "d:\\stock"
+
+    connect = sqlite3.connect(dest_dir + "/stock.db")
+    create_database(connect)
+
+    x = get_stk_code_name_list(MARKET.SH)
+    code_market_dict = {}
+    for v in x:
+        code_market_dict[v["code"]] = MARKET.SH
+    # print(code_market_dict)
+
+    em_import_block_to_sqlite(connect, code_market_dict)
+
+    connect.close()

hikyuu/data/hku_config_template.py

@@ -29,13 +29,15 @@
 quotation_server = {quotation_server}
 
 [block]
-type  = qianlong
-dir = {dir}/block
-指数板块 = zsbk.ini
-行业板块 = hybk.ini
-地域板块 = dybk.ini
-概念板块 = gnbk.ini
-self = self.ini
+type = sqlite3
+db = {dir}/stock.db
+;type  = qianlong
+;dir = {dir}/block
+;指数板块 = zsbk.ini
+;行业板块 = hybk.ini
+;地域板块 = dybk.ini
+;概念板块 = gnbk.ini
+;self = self.ini
 
 [preload]
 day = {day}
@@ -95,13 +97,11 @@
 quotation_server = {quotation_server}
 
 [block]
-type  = qianlong
-dir = {dir}/block
-指数板块 = zsbk.ini
-行业板块 = hybk.ini
-地域板块 = dybk.ini
-概念板块 = gnbk.ini
-self = self.ini
+type = mysql
+host = {host}
+port = {port}
+usr = {usr}
+pwd = {pwd}
 
 [preload]
 day = {day}

hikyuu/data/mysql_upgrade/0010.sql

@@ -0,0 +1,11 @@
+CREATE TABLE
+    IF NOT EXISTS `hku_base`.`block` (
+        `id` INT UNSIGNED NOT NULL AUTO_INCREMENT,
+        `category` VARCHAR(100) NOT NULL,
+        `name` VARCHAR(100) NOT NULL,
+        `market_code` VARCHAR(30) NOT NULL,
+        PRIMARY KEY (`id`),
+        INDEX `ix_block` (`category`, `name`)
+    ) COLLATE = 'utf8_general_ci' ENGINE = InnoDB;
+
+UPDATE `hku_base`.`version` set `version` = 10;

hikyuu/data/sqlite_upgrade/0011.sql

@@ -0,0 +1,16 @@
+BEGIN TRANSACTION;
+
+CREATE TABLE
+    IF NOT EXISTS `Block` (
+        "id" INTEGER,
+        `category` VARCHAR(100) NOT NULL,
+        `name` VARCHAR(100) NOT NULL,
+        `market_code` VARCHAR(30) NOT NULL,
+        PRIMARY KEY("id" AUTOINCREMENT)
+    );
+
+CREATE INDEX "ix_block" ON "block" (category, name);
+
+UPDATE `version` set `version` = 11;
+
+COMMIT;