{ "cells": [ { "cell_type": "code", "execution_count": null, "id": "f2e8d1b7-cbc5-4b02-8f3d-56b307a6dae8", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "from datetime import datetime\n", "from scipy import stats\n", "from sklearn.model_selection import TimeSeriesSplit\n", "\n", "def load_and_preprocess_data(file_path):\n", " \"\"\"加载并预处理指数数据\"\"\"\n", " try:\n", " df = pd.read_csv(file_path)\n", " \n", " # 将日期时间转换为datetime类型\n", " df['datetime'] = pd.to_datetime(df['date'] + ' ' + df['time'])\n", " \n", " # M1提取日期\n", " df['date_only'] = ___________________\n", " # M2提取小时\n", " df['hour'] = ___________________\n", " \n", " # 处理异常值\n", " # pct_change字段异常处理\n", " df['pct_change'] = pd.to_numeric(df['pct_change'], errors='coerce').fillna(0)\n", " \n", " return df\n", " \n", " except FileNotFoundError:\n", " print(\"错误：找不到CSV文件，请确保数据文件在正确路径下。\")\n", " return None\n", " except Exception as e:\n", " print(f\"发生未知错误：{e}\")\n", " return None\n", "\n", "def feature_engineering_1(df):\n", " \"\"\"特征1：指数在不同时间段的平均涨跌幅\"\"\"\n", " print(\"\\n开始构建特征1：指数在不同时间段的平均涨跌幅\")\n", " \n", " # 将一天划分为早盘/午盘/收盘前\n", " def time_period(hour):\n", " if 9 <= hour < 11:\n", " return 'morning'\n", " elif 11 <= hour < 14:\n", " return 'noon'\n", " else:\n", " return 'afternoon'\n", " \n", " df['time_period'] = df['hour'].apply(time_period)\n", " \n", " # M3计算指数在每个时间段的平均涨跌幅，可多行代码段实现\n", " period_index_change = ___________________\n", " _________________________________________\n", " _________________________________________\n", " print(f\"计算了 {len(period_index_change)} 个指数×日期的平均涨跌幅\")\n", " print(f\"平均涨跌幅: {period_index_change['avg_pct_change'].mean():.4f}\")\n", " \n", " return df, period_index_change\n", "\n", "def feature_engineering_2(df):\n", " \"\"\"特征2：板块 × 市场阶段的历史涨跌概率\"\"\"\n", " print(\"\\n开始构建特征2：板块 × 市场阶段的历史涨跌概率\")\n", " \n", " # 计算涨跌概率\n", " df['up'] = (df['pct_change'] > 0).astype(int)\n", " # M4 计算不同板块的指数在不同市场阶段的平均涨跌幅，可多行代码段实现\n", " sector_phase_probs = ___________________ \n", " _________________________________________\n", " _________________________________________\n", " print(f\"计算了 {len(sector_phase_probs)} 个板块×市场阶段的历史涨跌概率\")\n", " print(f\"平均涨跌概率: {sector_phase_probs['up_probability'].mean():.4f}\")\n", " \n", " return df, sector_phase_probs\n", "\n", "def feature_engineering_3(df):\n", " \"\"\"特征3：指数 × 板块的短期涨跌趋势偏好\"\"\"\n", " print(\"\\n开始构建特征3：指数 × 板块的短期涨跌趋势偏好\")\n", " \n", " # 使用过去5个交易日的涨跌趋势\n", " df['trend_5'] = df.groupby(['index_id'])['pct_change'].rolling(window=5).sum().reset_index(0,drop=True)\n", " # M5 计算不同板块的指数在过去5个交易日的涨跌趋势，可多行代码段实现\n", " index_sector_trend = ___________________ \n", " _________________________________________\n", " _________________________________________\n", " print(f\"计算了 {len(index_sector_trend)} 个指数×板块的短期涨跌趋势\")\n", " print(f\"平均趋势值: {index_sector_trend['avg_trend_5'].mean():.4f}\")\n", " \n", " return df, index_sector_trend\n", "\n", "def build_golden_test_set(df):\n", " \"\"\"构建黄金测试集并进行偏差检测\"\"\"\n", " print(\"\\n开始构建黄金测试集并进行偏差检测...\")\n", " \n", " df_sorted = df.sort_values('datetime')\n", " \n", " # M6 使用TimeSeriesSplit划分训练集和测试集，可多行代码段实现\n", " _________________________________________\n", " _________________________________________\n", " _________________________________________\n", " \n", " print(f\"训练集样本数: {len(train_data)}, 测试集样本数: {len(test_data)}\")\n", " \n", " # 偏差分析：不同板块涨跌概率分布\n", " print(\"\\n偏差分析：不同板块涨跌概率\")\n", " overall_probs = df.groupby('sector')['up'].mean()\n", " test_probs = test_data.groupby('sector')['up'].mean()\n", " \n", " all_sectors = sorted(set(df['sector'].unique()).union(set(test_data['sector'].unique())))\n", " overall_probs = overall_probs.reindex(all_sectors, fill_value=0)\n", " test_probs = test_probs.reindex(all_sectors, fill_value=0)\n", " \n", " # M7 KS检验\n", " _________________________________________\n", " print(f\"KS检验结果: statistic={ks_stat:.4f}, p-value={ks_p:.4f}\")\n", " \n", " return train_data, test_data\n", "\n", "def main():\n", " \"\"\"主函数，执行完整的数据处理流程\"\"\"\n", " \n", " df = load_and_preprocess_data('2.1.1.csv')\n", " if df is None:\n", " return\n", " \n", " # 特征工程\n", " df, period_index_change = feature_engineering_1(df)\n", " df, sector_phase_probs = feature_engineering_2(df)\n", " df, index_sector_trend = feature_engineering_3(df)\n", " \n", " # 保存高质量训练集\n", " high_quality_train = df[['index_id','date_only','time_period','pct_change','sector','market_phase']]\n", " high_quality_train.to_csv('high_quality_train_index.csv', index=False)\n", " print(\"\\n高质量训练集已保存为 high_quality_train_index.csv\")\n", " \n", " # 构建黄金测试集\n", " train_data, test_data = build_golden_test_set(df)\n", " \n", " print(\"\\n所有任务已完成！\")\n", "\n", "if __name__ == \"__main__\":\n", " main()\n" ] } ], "metadata": { "kernelspec": { "display_name": "Python [conda env:base] *", "language": "python", "name": "conda-base-py" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.7" } }, "nbformat": 4, "nbformat_minor": 5 }