fmda/read_and_clean_tutorial.ipynb

   1 {
   2  "cells": [
   3   {
   4    "cell_type": "code",
   5    "execution_count": null,
   6    "id": "9ddd1d89-abdb-4627-a0ca-23db006b62f4",
   7    "metadata": {},
   8    "outputs": [],
   9    "source": [
  10     "import yaml\n",
  11     "import pickle\n",
  12     "import os.path as osp\n",
  13     "import subprocess\n",
  14     "from datetime import timedelta\n",
  15     "from urllib.parse import urlparse\n",
  16     "import numpy as np\n",
  17     "import matplotlib.pyplot as plt\n",
  18     "from moisture_rnn_pkl import pkl2train\n",
  19     "import time\n",
  20     "from data_funcs import build_train_dict\n",
  21     "from utils import time_intp, str2time, filter_nan_values, read_pkl, read_yml, retrieve_url"
  22    ]
  23   },
  24   {
  25    "cell_type": "markdown",
  26    "id": "609ea544-ed92-40a6-892b-1943e9f6f620",
  27    "metadata": {},
  28    "source": [
  29     "## Setup"
  30    ]
  31   },
  32   {
  33    "cell_type": "code",
  34    "execution_count": null,
  35    "id": "41b0d403-7d6b-44f4-963f-8dc492ae0126",
  36    "metadata": {},
  37    "outputs": [],
  38    "source": [
  39     "retrieve_url(\"https://demo.openwfm.org/web/data/fmda/dicts/fmda_nw_202401-05_f05.pkl\", \"data/fmda_nw_202401-05_f05.pkl\")"
  40    ]
  41   },
  42   {
  43    "cell_type": "code",
  44    "execution_count": null,
  45    "id": "e69e37b9-73ef-45a1-9738-844f26dc3323",
  46    "metadata": {},
  47    "outputs": [],
  48    "source": [
  49     "params_data = read_yml(\"params_data.yaml\")\n",
  50     "params_data.update({'hours': 168})"
  51    ]
  52   },
  53   {
  54    "cell_type": "code",
  55    "execution_count": null,
  56    "id": "6b5c3c82-84ba-426c-b8d9-f540b5026158",
  57    "metadata": {},
  58    "outputs": [],
  59    "source": [
  60     "# dat = read_pkl(\"data/test_CA_202401.pkl\")\n",
  61     "dat = read_pkl(\"data/fmda_nw_202401-05_f05.pkl\")"
  62    ]
  63   },
  64   {
  65    "cell_type": "markdown",
  66    "id": "8afeae74-217e-41e5-a140-fd4df30e8148",
  67    "metadata": {},
  68    "source": [
  69     "## Format and Filter"
  70    ]
  71   },
  72   {
  73    "cell_type": "code",
  74    "execution_count": null,
  75    "id": "4003ced7-f08c-4cd3-a785-458c3588b235",
  76    "metadata": {
  77     "scrolled": true
  78    },
  79    "outputs": [],
  80    "source": [
  81     "train_h = build_train_dict([\"data/test_CA_202401.pkl\"], atm_source=\"HRRR\", params_data=params_data)"
  82    ]
  83   },
  84   {
  85    "cell_type": "code",
  86    "execution_count": null,
  87    "id": "7c47b87d-9d1d-4d85-aeb9-7248384ed16f",
  88    "metadata": {
  89     "scrolled": true
  90    },
  91    "outputs": [],
  92    "source": [
  93     "train_r = build_train_dict([\"data/test_CA_202401.pkl\"], atm_source=\"RAWS\", params_data=params_data, features_subset=['Ed', 'rain', 'wind'])"
  94    ]
  95   },
  96   {
  97    "cell_type": "code",
  98    "execution_count": null,
  99    "id": "cf33ee50-1b05-4f11-90fb-9cccf355fdd6",
 100    "metadata": {
 101     "scrolled": true
 102    },
 103    "outputs": [],
 104    "source": [
 105     "# Should Produce Informative Error\n",
 106     "train_r = build_train_dict([\"data/test_CA_202401.pkl\"], atm_source=\"RAWS\", params_data=params_data)"
 107    ]
 108   },
 109   {
 110    "cell_type": "code",
 111    "execution_count": null,
 112    "id": "aa0eb65f-a680-4630-beff-c01dabca32aa",
 113    "metadata": {},
 114    "outputs": [],
 115    "source": []
 116   },
 117   {
 118    "cell_type": "code",
 119    "execution_count": null,
 120    "id": "c2f20f9c-9d9a-488e-81ae-9aef54253541",
 121    "metadata": {},
 122    "outputs": [],
 123    "source": []
 124   }
 125  ],
 126  "metadata": {
 127   "kernelspec": {
 128    "display_name": "Python 3 (ipykernel)",
 129    "language": "python",
 130    "name": "python3"
 131   },
 132   "language_info": {
 133    "codemirror_mode": {
 134     "name": "ipython",
 135     "version": 3
 136    },
 137    "file_extension": ".py",
 138    "mimetype": "text/x-python",
 139    "name": "python",
 140    "nbconvert_exporter": "python",
 141    "pygments_lexer": "ipython3",
 142    "version": "3.12.5"
 143   }
 144  },
 145  "nbformat": 4,
 146  "nbformat_minor": 5
 147 }