fmda/OLD/create_RAWS_dict.ipynb

   1 {
   2  "cells": [
   3   {
   4    "cell_type": "markdown",
   5    "id": "b1100419-e2ec-46b6-89d3-327421288025",
   6    "metadata": {},
   7    "source": [
   8     "## Find Data Availability for Stations\n",
   9     "\n",
  10     "Find which RAWS stations in given state have data availability for variables of interest to fuel moisture model.\n",
  11     "\n",
  12     "See available Mesonet variables [here](https://developers.synopticdata.com/mesonet/v2/api-variables/)."
  13    ]
  14   },
  15   {
  16    "cell_type": "code",
  17    "execution_count": 1,
  18    "id": "edb47918-af48-4002-8a66-7e78c7d10c77",
  19    "metadata": {},
  20    "outputs": [],
  21    "source": [
  22     "# !pip install MesoPy\n",
  23     "import pickle\n",
  24     "from MesoPy import Meso\n",
  25     "import os.path as osp\n",
  26     "import os\n",
  27     "import pandas as pd\n",
  28     "import numpy as np\n",
  29     "\n",
  30     "outpath = \".\"\n",
  31     "\n",
  32     "meso_token=\"4192c18707b848299783d59a9317c6e1\" # Get your own token...\n",
  33     "m=Meso(meso_token)"
  34    ]
  35   },
  36   {
  37    "cell_type": "code",
  38    "execution_count": 2,
  39    "id": "fe4c6fbf-0242-4345-9fc5-38cdb5628861",
  40    "metadata": {},
  41    "outputs": [
  42     {
  43      "name": "stderr",
  44      "output_type": "stream",
  45      "text": [
  46       "C:\\Users\\jhirs\\anaconda3\\lib\\site-packages\\requests\\__init__.py:109: RequestsDependencyWarning: urllib3 (1.26.15) or chardet (5.1.0)/charset_normalizer (2.0.12) doesn't match a supported version!\n",
  47       "  warnings.warn(\n"
  48      ]
  49     }
  50    ],
  51    "source": [
  52     "os.chdir('..')\n",
  53     "from data_funcs import format_raws"
  54    ]
  55   },
  56   {
  57    "cell_type": "code",
  58    "execution_count": 3,
  59    "id": "37683794-406d-4524-99d3-2c167ff444ea",
  60    "metadata": {},
  61    "outputs": [],
  62    "source": [
  63     "time_start = \"202306010800\"  # June 1 2022 08:00 in format yyyymmddHHMM\n",
  64     "time_s2    = \"202306010900\"  # small time to get station ids\n",
  65     "time_end   = \"202306300900\"  # June 30 2022 09:00 in format yyyymmddHHMM\n",
  66     "state_str = \"CO\"\n",
  67     "\n",
  68     "vars='air_temp,relative_humidity,precip_accum,fuel_moisture,wind_speed,solar_radiation'"
  69    ]
  70   },
  71   {
  72    "cell_type": "code",
  73    "execution_count": 4,
  74    "id": "7dfb0387-6535-4976-9e9a-62e3cf9a69cd",
  75    "metadata": {},
  76    "outputs": [],
  77    "source": [
  78     "meso_obss = m.timeseries(start=time_start,end=time_s2, state=state_str, \n",
  79     "                             showemptystations = '0', vars=vars)"
  80    ]
  81   },
  82   {
  83    "cell_type": "code",
  84    "execution_count": 5,
  85    "id": "2255f323-2685-47b1-b862-f86eca6a3991",
  86    "metadata": {},
  87    "outputs": [],
  88    "source": [
  89     "station_df = pd.DataFrame(columns=['STID', 'air_temp', 'relative_humidity', 'precip_accum', 'fuel_moisture', 'wind_speed', 'solar_radiation'],\n",
  90     "                  index=range(0, len(meso_obss[\"STATION\"])))"
  91    ]
  92   },
  93   {
  94    "cell_type": "code",
  95    "execution_count": 6,
  96    "id": "6f4fbe0d-dd16-4c1c-950e-97e23e5a1918",
  97    "metadata": {},
  98    "outputs": [],
  99    "source": [
 100     "for i in range(0, station_df.shape[0]):\n",
 101     "    station_df[\"STID\"][i] = meso_obss[\"STATION\"][i][\"STID\"]\n",
 102     "    station_df[\"air_temp\"][i] = int(\"air_temp\" in meso_obss[\"STATION\"][i][\"SENSOR_VARIABLES\"].keys())\n",
 103     "    station_df[\"relative_humidity\"][i] = int(\"relative_humidity\" in meso_obss[\"STATION\"][i][\"SENSOR_VARIABLES\"].keys())\n",
 104     "    station_df[\"precip_accum\"][i] = int(\"precip_accum\" in meso_obss[\"STATION\"][i][\"SENSOR_VARIABLES\"].keys())\n",
 105     "    station_df[\"fuel_moisture\"][i] = int(\"fuel_moisture\" in meso_obss[\"STATION\"][i][\"SENSOR_VARIABLES\"].keys())\n",
 106     "    station_df[\"wind_speed\"][i] = int(\"wind_speed\" in meso_obss[\"STATION\"][i][\"SENSOR_VARIABLES\"].keys())\n",
 107     "    station_df[\"solar_radiation\"][i] = int(\"solar_radiation\" in meso_obss[\"STATION\"][i][\"SENSOR_VARIABLES\"].keys())"
 108    ]
 109   },
 110   {
 111    "cell_type": "code",
 112    "execution_count": 7,
 113    "id": "99938a78-4cac-440b-a2d8-6509b30f8c31",
 114    "metadata": {
 115     "tags": []
 116    },
 117    "outputs": [],
 118    "source": [
 119     "# Filter to stations with complete observations over time period\n",
 120     "station_df = station_df[\n",
 121     "    (station_df[\"fuel_moisture\"]==1) & \n",
 122     "    (station_df[\"relative_humidity\"]==1) &\n",
 123     "    (station_df[\"precip_accum\"]==1) &\n",
 124     "    (station_df[\"air_temp\"]==1) &\n",
 125     "    (station_df[\"wind_speed\"]==1) &\n",
 126     "    (station_df[\"solar_radiation\"]==1)\n",
 127     "]\n",
 128     "# Extract station IDs\n",
 129     "\n",
 130     "ids = station_df['STID'].tolist()"
 131    ]
 132   },
 133   {
 134    "cell_type": "code",
 135    "execution_count": 8,
 136    "id": "5492a962-b3da-4407-86d9-8ba35d004c29",
 137    "metadata": {
 138     "tags": []
 139    },
 140    "outputs": [
 141     {
 142      "name": "stdout",
 143      "output_type": "stream",
 144      "text": [
 145       "Number of RAWS Stations:  47\n"
 146      ]
 147     }
 148    ],
 149    "source": [
 150     "print('Number of RAWS Stations: ',station_df.shape[0])"
 151    ]
 152   },
 153   {
 154    "cell_type": "code",
 155    "execution_count": 9,
 156    "id": "99dde77e-5f6f-4e45-babf-f7599e8fca14",
 157    "metadata": {},
 158    "outputs": [],
 159    "source": [
 160     "# write output\n",
 161     "# station_df.to_csv(osp.join(outpath, 'station_df_co.csv'), index=False)"
 162    ]
 163   },
 164   {
 165    "cell_type": "markdown",
 166    "id": "1d92fcdb-afaa-4048-848e-e201b29040c4",
 167    "metadata": {},
 168    "source": [
 169     "## Get Observations"
 170    ]
 171   },
 172   {
 173    "cell_type": "code",
 174    "execution_count": 10,
 175    "id": "5ca1523d-b381-457c-9051-d2352144a666",
 176    "metadata": {
 177     "tags": []
 178    },
 179    "outputs": [],
 180    "source": [
 181     "# Queuery all stations with complete vars\n",
 182     "meso_ts = m.timeseries(time_start, time_end, stid=ids, showemptystations = '0', vars=vars)   # ask the object for data"
 183    ]
 184   },
 185   {
 186    "cell_type": "code",
 187    "execution_count": null,
 188    "id": "7abd402f-af58-46ce-8cb6-6854201f9d31",
 189    "metadata": {
 190     "tags": []
 191    },
 192    "outputs": [],
 193    "source": [
 194     "# Dictionary to be saved for testing\n",
 195     "test_dict = {}"
 196    ]
 197   },
 198   {
 199    "cell_type": "code",
 200    "execution_count": null,
 201    "id": "86e20fc8-f935-4577-83d7-ac188739698d",
 202    "metadata": {
 203     "tags": []
 204    },
 205    "outputs": [],
 206    "source": [
 207     "for i in range(0, len(meso_ts['STATION'])):\n",
 208     "    raws1 = format_raws(meso_ts['STATION'][i])\n",
 209     "    dict1={\n",
 210     "        'id': 'case'+str(i+1),\n",
 211     "        'time': raws1['time'],\n",
 212     "        'rain': raws1['rain'],\n",
 213     "        'fm' : raws1['fm'],\n",
 214     "        'rh' : raws1['rh'],\n",
 215     "        'temp' : raws1['temp'],\n",
 216     "        'Ed' : raws1['Ed'],\n",
 217     "        'Ew' : raws1['Ew'],\n",
 218     "        'wind' : raws1['wind_speed'],\n",
 219     "        'solar' : raws1['solar'],\n",
 220     "        'STID' : raws1['STID'],\n",
 221     "        'title' : 'RAWS Station '+raws1['STID'],\n",
 222     "        'descr' : 'RAWS sensor data, Colorado',\n",
 223     "        'hours':len(raws1['time']),\n",
 224     "        'h2':int(24*20),\n",
 225     "        'other': {'lon': raws1['lon'], 'lat': raws1['lat']}\n",
 226     "    }\n",
 227     "    test_dict['case'+str(i+1)] = dict1 # save to test dictionary"
 228    ]
 229   },
 230   {
 231    "cell_type": "markdown",
 232    "id": "35ab9bfc-1773-4d50-a83c-69780945245b",
 233    "metadata": {},
 234    "source": [
 235     "## Save Dictionary"
 236    ]
 237   },
 238   {
 239    "cell_type": "code",
 240    "execution_count": null,
 241    "id": "5f0f7cff-96bb-4b1e-a2b7-3fc2afcf5770",
 242    "metadata": {
 243     "tags": []
 244    },
 245    "outputs": [],
 246    "source": [
 247     "# Create file name from environment vars\n",
 248     "filename = \"testing_dict\"+\"_\"+state_str+\"_\"+time_start[0:6:1]\n",
 249     "print(filename)"
 250    ]
 251   },
 252   {
 253    "cell_type": "code",
 254    "execution_count": null,
 255    "id": "6aac2c25-b605-4437-b1de-f0f46cbba308",
 256    "metadata": {
 257     "tags": []
 258    },
 259    "outputs": [],
 260    "source": [
 261     "os.chdir('data')\n",
 262     "with open(filename+'.pickle', 'wb') as handle:\n",
 263     "    pickle.dump(test_dict, handle, protocol=pickle.HIGHEST_PROTOCOL)"
 264    ]
 265   },
 266   {
 267    "cell_type": "code",
 268    "execution_count": null,
 269    "id": "a6bdb681-0843-4dac-be53-e45aa0f4b991",
 270    "metadata": {},
 271    "outputs": [],
 272    "source": []
 273   }
 274  ],
 275  "metadata": {
 276   "kernelspec": {
 277    "display_name": "Python 3 (ipykernel)",
 278    "language": "python",
 279    "name": "python3"
 280   },
 281   "language_info": {
 282    "codemirror_mode": {
 283     "name": "ipython",
 284     "version": 3
 285    },
 286    "file_extension": ".py",
 287    "mimetype": "text/x-python",
 288    "name": "python",
 289    "nbconvert_exporter": "python",
 290    "pygments_lexer": "ipython3",
 291    "version": "3.9.12"
 292   }
 293  },
 294  "nbformat": 4,
 295  "nbformat_minor": 5
 296 }