{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# h5py update\n",
    "\n",
    "## HDF5 European Workshop for Science and Industry\n",
    "## ESRF, Grenoble, 2019-09-18"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# History, Particurals & Usage\n",
    "\n",
    " - Started in 2008 by Andrew Collette\n",
    "   - Now maintained by community\n",
    " - https://github.com/h5py/h5py\n",
    " - https://h5py.readthedocs.io/en/stable/\n",
    " - 129th most downlodaded package on pypi (mostil CI machines)\n",
    " - used by keras / tensorflow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# Basic Philosophy\n",
    "\n",
    " - Provides a \"pythonic\" wrapping of `libhdf5`\n",
    "   - less opnionated about use cases than `pytables`\n",
    "   - less tuned that `pytables`\n",
    " \n",
    "## Core Analogies\n",
    "\n",
    "- `dict` <-> {`h5py.File`, `h5py.Group`}\n",
    "  -  `g['key']` access to children (groups or datasets)\n",
    "- `np.array` <-> `h5py.Dataset`\n",
    "  - `Dateset` object support array protocol, slicing\n",
    "  - only pulls data from disk on demand"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# Write some data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: https://pypi.tuna.tsinghua.edu.cn/simple\n",
      "Collecting h5py\n",
      "  Downloading https://pypi.tuna.tsinghua.edu.cn/packages/6b/31/b5965f76e0bb2b02f273d87ec9cb59c77b9864ac27a0078c4229baa45dfc/h5py-3.10.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.8 MB)\n",
      "\u001b[K     |████████████████████████████████| 4.8 MB 3.0 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: numpy>=1.17.3 in /usr/local/lib/python3.8/dist-packages (from h5py) (1.24.4)\n",
      "Installing collected packages: h5py\n",
      "Successfully installed h5py-3.10.0\n"
     ]
    }
   ],
   "source": [
    "!pip install h5py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import h5py\n",
    "import numpy as np\n",
    "\n",
    "with h5py.File('example.h5', 'w') as fout:\n",
    "    # do the right thing in simple cases\n",
    "    fout['data'] = [0, 1, 2, 3, 4]\n",
    "    fout['nested/twoD'] = np.array([[1, 2], [3, 4]])\n",
    "    # method provides access to all of the dataset creation knobs\n",
    "    fout.create_dataset('data_B', \n",
    "                        data=np.arange(10).reshape(2, 5),\n",
    "                        chunks=(1, 5))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "# Read some data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<HDF5 file \"example.h5\" (mode r)>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fin = h5py.File('example.h5', 'r')\n",
    "# the File object\n",
    "fin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<HDF5 group \"/\" (3 members)>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# root group\n",
    "fin['/']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['data', 'data_B', 'nested']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(fin['/'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<HDF5 dataset \"data\": shape (5,), type \"<i8\">"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# a Dateset, has not read any data yet\n",
    "fin['data']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### numpy-stlye slicing on datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 2, 3, 4])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# pull data from disk to an array\n",
    "fin['data'][:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 2])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# pull part of the dataset\n",
    "fin['data'][1:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1, 3],\n",
       "       [6, 8]])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# handles numpy-style strided ND slicing\n",
    "fin['data_B'][:, 1::2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 3, 4])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# fancy slicing\n",
    "fin['data'][[0, 3, 4]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Accessing Nested Groups/Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<HDF5 dataset \"twoD\": shape (2, 2), type \"<i8\">"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# acess nested groups / datasets via repeated []\n",
    "fin['nested']['twoD']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<HDF5 dataset \"twoD\": shape (2, 2), type \"<i8\">"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Or use file-path like access\n",
    "fin['nested/twoD']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "### Close the file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Closed HDF5 file>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# if not using a context manager, remember to clean up!\n",
    "fin.close()\n",
    "fin"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# New is h5py 2.8\n",
    "\n",
    " - register new file drivers\n",
    " - track object creation order\n",
    " - lots of bug fixes!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# New in `h5py` 2.9\n",
    "\n",
    " - high-level API for creating virtual datasets\n",
    " - passing in python \"file-like\" objects to `h5py.File`\n",
    " - control chunk cache when creating `h5py.File`\n",
    " - `create_dataset_like` method\n",
    " - track creation order of attributes\n",
    " - bug fixes!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "## High level API for Virtual Datasets \n",
    "\n",
    "\n",
    "- Work stared by Aaron Parsons at DLS\n",
    "- continued by Thomas Caswell at NSLS-II\n",
    "- finished by Thomas Kluyver at EuXFEL\n",
    "\n",
    "\n",
    "low-level API has been availble from h5py 2.6"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Create some data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# create some sample data\n",
    "data = np.arange(0, 100).reshape(1, 100) + np.arange(1, 5).reshape(4, 1)\n",
    "\n",
    "# Create source files (0.h5 to 3.h5)\n",
    "for n in range(4):\n",
    "    with h5py.File(f\"{n}.h5\", \"w\") as f:\n",
    "        d = f.create_dataset(\"data\", (100,), \"i4\", data[n])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "### Create the Virtual Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Assemble virtual dataset\n",
    "layout = h5py.VirtualLayout(shape=(4, 100), dtype=\"i4\")\n",
    "for n in range(4):\n",
    "    layout[n] = h5py.VirtualSource(f\"{n}.h5\", \"data\", shape=(100,))\n",
    "\n",
    "# Add virtual dataset to output file\n",
    "with h5py.File(\"VDS.h5\", \"w\", libver=\"latest\") as f:\n",
    "    # the virtual dataset\n",
    "    f.create_virtual_dataset(\"data_A\", layout, fillvalue=-5)\n",
    "    # normal dataset with identical values\n",
    "    f.create_dataset(\"data_B\", data=data, dtype='i4')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Read it back"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Virtual dataset: <HDF5 dataset \"data_A\": shape (4, 100), type \"<i4\">\n",
      "[[ 1 11 21 31 41 51 61 71 81 91]\n",
      " [ 2 12 22 32 42 52 62 72 82 92]\n",
      " [ 3 13 23 33 43 53 63 73 83 93]\n",
      " [ 4 14 24 34 44 54 64 74 84 94]]\n",
      "Normal dataset : <HDF5 dataset \"data_B\": shape (4, 100), type \"<i4\">\n",
      "[[ 1 11 21 31 41 51 61 71 81 91]\n",
      " [ 2 12 22 32 42 52 62 72 82 92]\n",
      " [ 3 13 23 33 43 53 63 73 83 93]\n",
      " [ 4 14 24 34 44 54 64 74 84 94]]\n"
     ]
    }
   ],
   "source": [
    "# read data back\n",
    "# virtual dataset is transparent for reader!\n",
    "with h5py.File(\"VDS.h5\", \"r\") as f:\n",
    "    print(f\"Virtual dataset: {f['data_A']}\")\n",
    "    print(f[\"data_A\"][:, ::10])\n",
    "    print(f\"Normal dataset : {f['data_B']}\")\n",
    "    print(f[\"data_B\"][:, ::10])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "## Pass Python file-like objects to `h5py.File`\n",
    "\n",
    " - contributed by Andrey Paramonov (Андрей Парамонов)\n",
    " - can pass in object returned by `open` or a `BytesIO` object"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Creat a `BtyesIO` object and write data to it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "from io import BytesIO\n",
    "\n",
    "obj = BytesIO()\n",
    "with h5py.File(obj, 'w') as fout:\n",
    "    fout['data'] = np.linspace(0, 30, 10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "### Read the data back"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the frist 5 bytse: b'\\x89HDF\\r'\n"
     ]
    }
   ],
   "source": [
    "obj.seek(0)\n",
    "print(f\"the frist 5 bytse: {obj.read(5)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<HDF5 dataset \"data\": shape (10,), type \"<f8\">\n"
     ]
    }
   ],
   "source": [
    "obj.seek(0)\n",
    "with h5py.File(obj, 'r') as fin:\n",
    "    print(fin['data'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Write buffer to disk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "obj.seek(0)\n",
    "with open('test_out.h5', 'wb') as fout:\n",
    "    fout.write(obj.getbuffer())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "### Read back with hdf5 opening the file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<HDF5 dataset \"data\": shape (10,), type \"<f8\">\n"
     ]
    }
   ],
   "source": [
    "with h5py.File('test_out.h5', 'r') as fin:\n",
    "    print(fin['data'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "### Use `open` to read the file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<HDF5 dataset \"data\": shape (10,), type \"<f8\">\n"
     ]
    }
   ],
   "source": [
    "with open('test_out.h5', 'rb') as raw_file:\n",
    "    with h5py.File(raw_file, 'r') as fin:\n",
    "        print(fin['data'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "## Better KeysView repr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<KeysViewHDF5 ['data', 'data_B', 'nested']>\n"
     ]
    }
   ],
   "source": [
    "with h5py.File('example.h5', 'r') as fin:\n",
    "    print(fin.keys())\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# New in h5py 2.10\n",
    "\n",
    "- Better support for reading bit fields\n",
    "- deprecate implicit file mode\n",
    "- better tab-completion out-of-the-box in IPython\n",
    "- add `Dataset.make_scale` helper\n",
    "- improve handling of spcial data types\n",
    "- expose `H5PL` functions\n",
    "- expose `H5Dread_chunk` and `h5d.read_direct_chunk`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "## Require file mode (so we can change the default next release)\n",
    "\n",
    "- the current default mode is \"open append, or create if needed\"\n",
    "- this is dangerous as users may accindentally mutate files they did not want to!\n",
    "- does not match behivor of `open`\n",
    "- for back-compatibliity did not want to change default in one step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/tcaswell/mc3/envs/dd37/lib/python3.7/site-packages/ipykernel_launcher.py:1: H5pyDeprecationWarning: The default file mode will change to 'r' (read-only) in h5py 3.0. To suppress this warning, pass the mode you need to h5py.File(), or set the global default h5.get_config().default_file_mode, or set the environment variable H5PY_DEFAULT_READONLY=1. Available modes are: 'r', 'r+', 'w', 'w-'/'x', 'a'. See the docs for details.\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "with h5py.File('blahblah.h5') as fout:\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "h5py.get_config().default_file_mode = 'r'\n",
    "with h5py.File('blahblah.h5') as fout:\n",
    "    pass\n",
    "# put it back to default just to be tidy!\n",
    "h5py.get_config().default_file_mode = None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "## `make_scale` helper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "with h5py.File(\"with_scale.h5\", 'w') as fout:\n",
    "    fout['data'] = range(10)\n",
    "    fout['pos'] = np.arange(10) + 5\n",
    "    fout['pos'].make_scale(\"pos\")\n",
    "    fout['data'].dims[0].attach_scale(fout['pos'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HDF5 \"with_scale.h5\" {\n",
      "DATASET \"data\" {\n",
      "   DATATYPE  H5T_STD_I64LE\n",
      "   DATASPACE  SIMPLE { ( 10 ) / ( 10 ) }\n",
      "   DATA {\n",
      "   (0): 0, 1, 2, 3, 4, 5, 6, 7, 8, 9\n",
      "   }\n",
      "   ATTRIBUTE \"DIMENSION_LIST\" {\n",
      "      DATATYPE  H5T_VLEN { H5T_REFERENCE { H5T_STD_REF_OBJECT }}\n",
      "      DATASPACE  SIMPLE { ( 1 ) / ( 1 ) }\n",
      "      DATA {\n",
      "      (0): (DATASET 1400 /pos )\n",
      "      }\n",
      "   }\n",
      "}\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "!h5dump --dataset=data with_scale.h5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    },
    "user_expressions": []
   },
   "source": [
    "# 行情数据存储例子\n",
    "\n",
    "以下部分由大富翁量化课程提供，获取行情的代码需要在课程中运行"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-01-01 18:09:59,853 I 2537 cfg4py.core:update_config:280 | configuration is\n",
      "alpha: {data_home: ~/zillionare/alpha/data, tts_server: 'http://127.0.0.1:5002/api/tts?'}\n",
      "backtest: {url: 'http://192.168.100.114:7080/backtest/api/trade/v0.5/'}\n",
      "influxdb: {bucket_name: zillionare, enable_compress: true, max_query_size: 5000, org: zillionare,\n",
      "  token: hwxHycJfp_t6bCOYe2MhEDW4QBOO4FDtgeBWnPR6bGZJGEZ_41m_OHtTJFZKyD2HsbVqkZM8rJNkMvjyoXCG6Q==,\n",
      "  url: 'http://192.168.100.101:58086'}\n",
      "notify: {dingtalk_access_token: 58df072143b52368086736cb38236753073ccde6537650cad1d5567747803563,\n",
      "  keyword: trader}\n",
      "pluto: {store: ~/zillionare/pluto/store}\n",
      "redis: {dsn: 'redis://192.168.100.101:56379'}\n",
      "tasks: {pooling: false, wr: false}\n",
      "\n",
      "2024-01-01 18:09:59,856 I 2537 /usr/local/lib/python3.8/dist-packages/omicron/dal/cache.py:init:94 | init redis cache...\n",
      "2024-01-01 18:09:59,866 I 2537 /usr/local/lib/python3.8/dist-packages/omicron/dal/cache.py:init:124 | redis cache is inited\n",
      "2024-01-01 18:09:59,976 I 2537 omicron.models.security:load_securities:333 | 7037 securities loaded, types: {'fjm', 'stock', 'lof', 'etf', 'mmf', 'reits', 'index'}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "init securities done\n"
     ]
    }
   ],
   "source": [
    "from coursea import *\n",
    "await init()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "codes = [\"000001.XSHE\", \"600000.XSHG\"]\n",
    "\n",
    "h5file = \"/tmp/bars.h5\"\n",
    "\n",
    "h5 = h5py.File(h5file, \"a\")\n",
    "\n",
    "if \"1m\" not in h5.keys():\n",
    "    h5.create_group(\"/1m\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "├── 1d\n",
      "│   ├── 000001.XSHE (2400000)\n",
      "│   └── 600000.XSHG (2400000)\n",
      "└── 1m\n"
     ]
    }
   ],
   "source": [
    "def convert_frame(bars):\n",
    "    # h5 不能处理 np.datetime64，转换成整数\n",
    "    dtype = bars.dtype.descr\n",
    "    dtype[0] = ('frame', 'i8')\n",
    "    \n",
    "    return bars.astype(dtype)\n",
    "   \n",
    "def append_ds(name: str, bars):\n",
    "    ds = h5.get(name)\n",
    "    if ds is None:\n",
    "        ds = h5.create_dataset(name, data = bars, chunks=True, maxshape=(None,))\n",
    "    else:\n",
    "        nold = ds.shape[0]\n",
    "        nnew = len(bars)\n",
    "        ds.resize(nold + nnew, axis=0)\n",
    "        ds[-nnew:] = bars\n",
    "        \n",
    "    return ds\n",
    "\n",
    "# 每日增加行情数据\n",
    "async def save_bars(codes:List[str], ft: FrameType):    \n",
    "    for code in codes:\n",
    "        bars = await Stock.get_bars(code, 240, ft)\n",
    "        append_ds(f\"/{ft.value}/{code}\", convert_frame(bars))\n",
    "\n",
    "\n",
    "    \n",
    "# 显示h5文件结构\n",
    "\n",
    "def h5_tree(val, pre=''):\n",
    "    items = len(val)\n",
    "    for key, val in val.items():\n",
    "        items -= 1\n",
    "        if items == 0:\n",
    "            # the last item\n",
    "            if type(val) == h5py._hl.group.Group:\n",
    "                print(pre + '└── ' + key)\n",
    "                h5_tree(val, pre+'    ')\n",
    "            else:\n",
    "                print(pre + '└── ' + key + ' (%d)' % len(val))\n",
    "        else:\n",
    "            if type(val) == h5py._hl.group.Group:\n",
    "                print(pre + '├── ' + key)\n",
    "                h5_tree(val, pre+'│   ')\n",
    "            else:\n",
    "                print(pre + '├── ' + key + ' (%d)' % len(val))\n",
    "                \n",
    "h5_tree(h5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    },
    {
     "ename": "CancelledError",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mCancelledError\u001b[0m                            Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[141], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[38;5;241m10000\u001b[39m):\n\u001b[0;32m----> 2\u001b[0m     \u001b[38;5;28;01mawait\u001b[39;00m save_bars(codes, FrameType\u001b[38;5;241m.\u001b[39mDAY)\n\u001b[1;32m      3\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m i \u001b[38;5;241m%\u001b[39m \u001b[38;5;241m1000\u001b[39m \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m      4\u001b[0m         \u001b[38;5;28mprint\u001b[39m(i)\n",
      "Cell \u001b[0;32mIn[140], line 23\u001b[0m, in \u001b[0;36msave_bars\u001b[0;34m(codes, ft)\u001b[0m\n\u001b[1;32m     21\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21msave_bars\u001b[39m(codes:List[\u001b[38;5;28mstr\u001b[39m], ft: FrameType):    \n\u001b[1;32m     22\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m code \u001b[38;5;129;01min\u001b[39;00m codes:\n\u001b[0;32m---> 23\u001b[0m         bars \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m Stock\u001b[38;5;241m.\u001b[39mget_bars(code, \u001b[38;5;241m240\u001b[39m, ft)\n\u001b[1;32m     24\u001b[0m         append_ds(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m/\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mft\u001b[38;5;241m.\u001b[39mvalue\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m/\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mcode\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m, convert_frame(bars))\n",
      "File \u001b[0;32m/usr/local/lib/python3.8/dist-packages/omicron/models/stock.py:471\u001b[0m, in \u001b[0;36mStock.get_bars\u001b[0;34m(cls, code, n, frame_type, end, fq, unclosed)\u001b[0m\n\u001b[1;32m    468\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    469\u001b[0m     end0 \u001b[38;5;241m=\u001b[39m end\n\u001b[0;32m--> 471\u001b[0m bars \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39m_get_persisted_bars_n(code, frame_type, n0, end0)\n\u001b[1;32m    472\u001b[0m merged \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mconcatenate((bars, cached))\n\u001b[1;32m    473\u001b[0m bars \u001b[38;5;241m=\u001b[39m merged[\u001b[38;5;241m-\u001b[39mn:]\n",
      "File \u001b[0;32m/usr/local/lib/python3.8/dist-packages/omicron/models/stock.py:610\u001b[0m, in \u001b[0;36mStock._get_persisted_bars_n\u001b[0;34m(cls, code, frame_type, n, end)\u001b[0m\n\u001b[1;32m    586\u001b[0m serializer \u001b[38;5;241m=\u001b[39m DataframeDeserializer(\n\u001b[1;32m    587\u001b[0m     encoding\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mutf-8\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    588\u001b[0m     names\u001b[38;5;241m=\u001b[39m[\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    606\u001b[0m     parse_dates\u001b[38;5;241m=\u001b[39m[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mframe\u001b[39m\u001b[38;5;124m\"\u001b[39m],\n\u001b[1;32m    607\u001b[0m )\n\u001b[1;32m    609\u001b[0m client \u001b[38;5;241m=\u001b[39m get_influx_client()\n\u001b[0;32m--> 610\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m client\u001b[38;5;241m.\u001b[39mquery(flux, serializer)\n\u001b[1;32m    611\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m result\u001b[38;5;241m.\u001b[39mto_records(index\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)\u001b[38;5;241m.\u001b[39mastype(bars_dtype)\n",
      "File \u001b[0;32m/usr/local/lib/python3.8/dist-packages/omicron/dal/influx/influxclient.py:217\u001b[0m, in \u001b[0;36mInfluxClient.query\u001b[0;34m(self, flux, deserializer)\u001b[0m\n\u001b[1;32m    214\u001b[0m     flux \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mstr\u001b[39m(flux)\n\u001b[1;32m    216\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mwith\u001b[39;00m ClientSession() \u001b[38;5;28;01mas\u001b[39;00m session:\n\u001b[0;32m--> 217\u001b[0m     \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mwith\u001b[39;00m session\u001b[38;5;241m.\u001b[39mpost(\n\u001b[1;32m    218\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_query_url, data\u001b[38;5;241m=\u001b[39mflux, headers\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_query_headers\n\u001b[1;32m    219\u001b[0m     ) \u001b[38;5;28;01mas\u001b[39;00m resp:\n\u001b[1;32m    220\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m resp\u001b[38;5;241m.\u001b[39mstatus \u001b[38;5;241m!=\u001b[39m \u001b[38;5;241m200\u001b[39m:\n\u001b[1;32m    221\u001b[0m             err \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m resp\u001b[38;5;241m.\u001b[39mjson()\n",
      "File \u001b[0;32m/usr/local/lib/python3.8/dist-packages/aiohttp/client.py:1141\u001b[0m, in \u001b[0;36m_BaseRequestContextManager.__aenter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1140\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__aenter__\u001b[39m(\u001b[38;5;28mself\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m _RetType:\n\u001b[0;32m-> 1141\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_resp \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_coro\n\u001b[1;32m   1142\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_resp\n",
      "File \u001b[0;32m/usr/local/lib/python3.8/dist-packages/aiohttp/client.py:560\u001b[0m, in \u001b[0;36mClientSession._request\u001b[0;34m(self, method, str_or_url, params, data, json, cookies, headers, skip_auto_headers, auth, allow_redirects, max_redirects, compress, chunked, expect100, raise_for_status, read_until_eof, proxy, proxy_auth, timeout, verify_ssl, fingerprint, ssl_context, ssl, proxy_headers, trace_request_ctx, read_bufsize)\u001b[0m\n\u001b[1;32m    558\u001b[0m resp \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m req\u001b[38;5;241m.\u001b[39msend(conn)\n\u001b[1;32m    559\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 560\u001b[0m     \u001b[38;5;28;01mawait\u001b[39;00m resp\u001b[38;5;241m.\u001b[39mstart(conn)\n\u001b[1;32m    561\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m:\n\u001b[1;32m    562\u001b[0m     resp\u001b[38;5;241m.\u001b[39mclose()\n",
      "File \u001b[0;32m/usr/local/lib/python3.8/dist-packages/aiohttp/client_reqrep.py:899\u001b[0m, in \u001b[0;36mClientResponse.start\u001b[0;34m(self, connection)\u001b[0m\n\u001b[1;32m    897\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m    898\u001b[0m     protocol \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_protocol\n\u001b[0;32m--> 899\u001b[0m     message, payload \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m protocol\u001b[38;5;241m.\u001b[39mread()  \u001b[38;5;66;03m# type: ignore[union-attr]\u001b[39;00m\n\u001b[1;32m    900\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m http\u001b[38;5;241m.\u001b[39mHttpProcessingError \u001b[38;5;28;01mas\u001b[39;00m exc:\n\u001b[1;32m    901\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m ClientResponseError(\n\u001b[1;32m    902\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrequest_info,\n\u001b[1;32m    903\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhistory,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    906\u001b[0m         headers\u001b[38;5;241m=\u001b[39mexc\u001b[38;5;241m.\u001b[39mheaders,\n\u001b[1;32m    907\u001b[0m     ) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mexc\u001b[39;00m\n",
      "File \u001b[0;32m/usr/local/lib/python3.8/dist-packages/aiohttp/streams.py:616\u001b[0m, in \u001b[0;36mDataQueue.read\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    614\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_waiter \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_loop\u001b[38;5;241m.\u001b[39mcreate_future()\n\u001b[1;32m    615\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 616\u001b[0m     \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_waiter\n\u001b[1;32m    617\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (asyncio\u001b[38;5;241m.\u001b[39mCancelledError, asyncio\u001b[38;5;241m.\u001b[39mTimeoutError):\n\u001b[1;32m    618\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_waiter \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "\u001b[0;31mCancelledError\u001b[0m: "
     ]
    }
   ],
   "source": [
    "await save_bars(codes, FrameType.DAY)\n",
    "await save_bars(code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2400000"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mbars = h5[\"1d\"][\"000001.XSHE\"]\n",
    "\n",
    "tm = datetime.datetime(2023, 12, 29, 14, 58)\n",
    "len(mbars[mbars[\"frame\"]>tm.timestamp()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "celltoolbar": "Slideshow",
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}