| blob | 09dc0a568ca05558b2be60b09309ba70037e42eb |
1 {
2 "cells": [
3 {
4 "cell_type": "markdown",
5 "id": "244c2fb0-4339-476c-a2db-a641e124e25a",
6 "metadata": {},
7 "source": [
8 "# v2.1 exploration trying to make it work better"
9 ]
10 },
11 {
12 "cell_type": "code",
13 "execution_count": null,
14 "id": "e6cc7920-e380-4b81-bac0-cd6840450e9a",
15 "metadata": {},
16 "outputs": [],
17 "source": [
18 "# Environment\n",
19 "import os\n",
20 "import os.path as osp\n",
21 "import numpy as np\n",
22 "import pandas as pd\n",
23 "import tensorflow as tf\n",
24 "import matplotlib.pyplot as plt\n",
25 "import sys\n",
26 "# Local modules\n",
27 "sys.path.append('..')\n",
28 "import reproducibility\n",
29 "import pandas as pd\n",
30 "from utils import print_dict_summary\n",
31 "from data_funcs import rmse, build_train_dict, combine_nested, subset_by_features\n",
32 "from moisture_rnn import RNNParams, RNNData, RNN, RNN_LSTM, rnn_data_wrap\n",
33 "from moisture_rnn_pkl import pkl2train\n",
34 "from tensorflow.keras.callbacks import Callback\n",
35 "from utils import hash2\n",
36 "import copy\n",
37 "import logging\n",
38 "import pickle\n",
39 "from utils import logging_setup, read_yml, read_pkl, hash_ndarray, hash_weights, str2time\n",
40 "import yaml\n",
41 "import copy\n",
42 "import time"
43 ]
44 },
45 {
46 "cell_type": "code",
47 "execution_count": null,
48 "id": "f58e8839-bf0e-4995-b966-c09e4df001ce",
49 "metadata": {},
50 "outputs": [],
51 "source": [
52 "logging_setup()"
53 ]
54 },
55 {
56 "cell_type": "markdown",
57 "id": "fae67b50-f916-45a7-bcc7-61995ba39449",
58 "metadata": {},
59 "source": [
60 "## Test Data"
61 ]
62 },
63 {
64 "cell_type": "code",
65 "execution_count": null,
66 "id": "3efed1fa-9cda-4934-8a6c-edcf179c8755",
67 "metadata": {},
68 "outputs": [],
69 "source": [
70 "file_paths = ['data/fmda_rocky_202403-05_f05.pkl']"
71 ]
72 },
73 {
74 "cell_type": "code",
75 "execution_count": null,
76 "id": "28fd3746-1861-4afa-ab7e-ac449fbed322",
77 "metadata": {},
78 "outputs": [],
79 "source": [
80 "# Params used for data filtering\n",
81 "params_data = read_yml(\"params_data.yaml\") \n",
82 "params_data"
83 ]
84 },
85 {
86 "cell_type": "code",
87 "execution_count": null,
88 "id": "32a46674-8377-47f8-9c3a-e6b07f9505cf",
89 "metadata": {},
90 "outputs": [],
91 "source": [
92 "params = read_yml(\"params.yaml\", subkey='rnn') \n",
93 "params = RNNParams(params)\n",
94 "params.update({'epochs': 200, \n",
95 " 'learning_rate': 0.001,\n",
96 " 'activation': ['tanh', 'tanh'], # Activation for RNN Layers, Dense layers respectively.\n",
97 " 'recurrent_layers': 2, 'recurrent_units': 30, \n",
98 " 'dense_layers': 2, 'dense_units': 30,\n",
99 " 'early_stopping_patience': 30, # how many epochs of no validation accuracy gain to wait before stopping\n",
100 " 'batch_schedule_type': 'exp', # Hidden state batch reset schedule\n",
101 " 'bmin': 20, # Lower bound of hidden state batch reset, \n",
102 " 'bmax': params_data['hours'], # Upper bound of hidden state batch reset, using max hours\n",
103 " 'features_list': ['Ed', 'Ew', 'rain', 'elev', 'lon', 'lat', 'solar', 'wind'],\n",
104 " 'timesteps': 12\n",
105 " })"
106 ]
107 },
108 {
109 "cell_type": "code",
110 "execution_count": null,
111 "id": "c45cb8ef-41fc-4bf7-b506-dad5fd24abb3",
112 "metadata": {},
113 "outputs": [],
114 "source": [
115 "dat = read_pkl(file_paths[0])"
116 ]
117 },
118 {
119 "cell_type": "code",
120 "execution_count": null,
121 "id": "3c960d69-4f8a-4abb-a5d9-ed6cf98f899b",
122 "metadata": {},
123 "outputs": [],
124 "source": [
125 "import importlib\n",
126 "import data_funcs\n",
127 "importlib.reload(data_funcs)\n",
128 "from data_funcs import build_train_dict"
129 ]
130 },
131 {
132 "cell_type": "code",
133 "execution_count": null,
134 "id": "369cd913-85cb-4855-a80c-817d84637852",
135 "metadata": {},
136 "outputs": [],
137 "source": [
138 "params_data.update({'hours': None})"
139 ]
140 },
141 {
142 "cell_type": "code",
143 "execution_count": null,
144 "id": "8cdc2ce8-45b4-4caa-81d9-646271ff2e97",
145 "metadata": {
146 "scrolled": true
147 },
148 "outputs": [],
149 "source": [
150 "train3 = build_train_dict(file_paths, params_data, spatial=False, forecast_step=3, drop_na=True)\n"
151 ]
152 },
153 {
154 "cell_type": "code",
155 "execution_count": null,
156 "id": "3c4548ae-caa4-4bc4-9122-9f24e7e59ef7",
157 "metadata": {},
158 "outputs": [],
159 "source": []
160 },
161 {
162 "cell_type": "code",
163 "execution_count": null,
164 "id": "3dbb6f24-4435-47b3-90c6-6176582b0d4c",
165 "metadata": {},
166 "outputs": [],
167 "source": []
168 },
169 {
170 "cell_type": "markdown",
171 "id": "6322f0bc-107d-40a5-96dc-804495085a99",
172 "metadata": {
173 "jp-MarkdownHeadingCollapsed": true
174 },
175 "source": [
176 "## Test Other ML"
177 ]
178 },
179 {
180 "cell_type": "code",
181 "execution_count": null,
182 "id": "12992b9a-407f-4131-ac61-e1dc338386bf",
183 "metadata": {},
184 "outputs": [],
185 "source": [
186 "params = read_yml(\"params.yaml\", subkey='xgb')\n",
187 "params"
188 ]
189 },
190 {
191 "cell_type": "code",
192 "execution_count": null,
193 "id": "f214fdf8-bb76-4912-8f8c-5d0c8c1230c2",
194 "metadata": {},
195 "outputs": [],
196 "source": [
197 "dat = read_pkl(\"data/train.pkl\")"
198 ]
199 },
200 {
201 "cell_type": "code",
202 "execution_count": null,
203 "id": "888b7805-15f6-4c09-a05b-7aed7d253f6e",
204 "metadata": {},
205 "outputs": [],
206 "source": [
207 "cases = [*dat.keys()]"
208 ]
209 },
210 {
211 "cell_type": "code",
212 "execution_count": null,
213 "id": "375055d8-c070-4639-9561-e47d3f21f1f8",
214 "metadata": {},
215 "outputs": [],
216 "source": [
217 "rnn_dat = RNNData(dat[cases[10]], params['scaler'], params['features_list'])\n",
218 "rnn_dat.train_test_split(\n",
219 " time_fracs = [.8, .1, .1]\n",
220 ")\n",
221 "rnn_dat.scale_data()"
222 ]
223 },
224 {
225 "cell_type": "code",
226 "execution_count": null,
227 "id": "e79f8dc8-5cf8-4190-b4ff-e640f61bd78b",
228 "metadata": {},
229 "outputs": [],
230 "source": [
231 "from moisture_models import XGB, RF, LM"
232 ]
233 },
234 {
235 "cell_type": "code",
236 "execution_count": null,
237 "id": "b3aeb47f-261e-4e29-9eeb-67215e5628f6",
238 "metadata": {},
239 "outputs": [],
240 "source": [
241 "mod = XGB(params)"
242 ]
243 },
244 {
245 "cell_type": "code",
246 "execution_count": null,
247 "id": "cae9a20d-1caf-45aa-a9c4-aef21b65d9c8",
248 "metadata": {},
249 "outputs": [],
250 "source": [
251 "mod.params"
252 ]
253 },
254 {
255 "cell_type": "code",
256 "execution_count": null,
257 "id": "68a07b25-c586-4fc4-a3d5-c857354e7a2c",
258 "metadata": {},
259 "outputs": [],
260 "source": [
261 "mod.fit(rnn_dat.X_train, rnn_dat.y_train)"
262 ]
263 },
264 {
265 "cell_type": "code",
266 "execution_count": null,
267 "id": "c8f88819-0a7a-4420-abb9-56a47015a4de",
268 "metadata": {},
269 "outputs": [],
270 "source": [
271 "preds = mod.predict(rnn_dat.X_test)"
272 ]
273 },
274 {
275 "cell_type": "code",
276 "execution_count": null,
277 "id": "cb7cdf14-74d6-45e4-bc1b-7d4d47dd41ac",
278 "metadata": {},
279 "outputs": [],
280 "source": [
281 "rmse(preds, rnn_dat.y_test)"
282 ]
283 },
284 {
285 "cell_type": "code",
286 "execution_count": null,
287 "id": "74d478c7-8c01-448e-9a00-dd0e1ee8e325",
288 "metadata": {},
289 "outputs": [],
290 "source": [
291 "plt.plot(rnn_dat.y_test)\n",
292 "plt.plot(preds)"
293 ]
294 },
295 {
296 "cell_type": "code",
297 "execution_count": null,
298 "id": "c5441014-c39a-4414-a779-95b81e1ed6a8",
299 "metadata": {},
300 "outputs": [],
301 "source": [
302 "params = read_yml(\"params.yaml\", subkey='rf')\n",
303 "rnn_dat = RNNData(dat[cases[10]], features_list = ['Ed', 'Ew', 'solar', 'wind', 'rain'])\n",
304 "rnn_dat.train_test_split(\n",
305 " time_fracs = [.8, .1, .1]\n",
306 ")"
307 ]
308 },
309 {
310 "cell_type": "code",
311 "execution_count": null,
312 "id": "cafe711a-20cb-4bd3-a4bc-4995a843a021",
313 "metadata": {},
314 "outputs": [],
315 "source": [
316 "import importlib\n",
317 "import moisture_models\n",
318 "importlib.reload(moisture_models)"
319 ]
320 },
321 {
322 "cell_type": "code",
323 "execution_count": null,
324 "id": "ee45f7d6-f57f-4ff6-995a-527565565f94",
325 "metadata": {},
326 "outputs": [],
327 "source": [
328 "params"
329 ]
330 },
331 {
332 "cell_type": "code",
333 "execution_count": null,
334 "id": "fafe76e5-0212-4bd1-a058-535935a08780",
335 "metadata": {},
336 "outputs": [],
337 "source": [
338 "mod2 = RF(params)\n",
339 "mod2.fit(rnn_dat.X_train, rnn_dat.y_train.flatten())\n",
340 "preds2 = mod2.predict(rnn_dat.X_test)\n",
341 "print(rmse(preds2, rnn_dat.y_test.flatten()))\n",
342 "plt.plot(rnn_dat.y_test)\n",
343 "plt.plot(preds2)"
344 ]
345 },
346 {
347 "cell_type": "code",
348 "execution_count": null,
349 "id": "c0ab4244-996c-49af-bf4a-8b0c47b0b6db",
350 "metadata": {},
351 "outputs": [],
352 "source": [
353 "from moisture_models import RF\n",
354 "mod2 = RF(params)"
355 ]
356 },
357 {
358 "cell_type": "code",
359 "execution_count": null,
360 "id": "aa6c33fd-db35-4c77-9eee-fdb39a934959",
361 "metadata": {},
362 "outputs": [],
363 "source": []
364 },
365 {
366 "cell_type": "code",
367 "execution_count": null,
368 "id": "c5598bfe-2d87-4d23-869e-aff127782462",
369 "metadata": {},
370 "outputs": [],
371 "source": [
372 "params = read_yml(\"params.yaml\", subkey='lm')\n",
373 "rnn_dat = RNNData(dat[cases[10]], features_list = ['Ed', 'Ew', 'solar', 'wind', 'rain'])\n",
374 "rnn_dat.train_test_split(\n",
375 " time_fracs = [.8, .1, .1]\n",
376 ")\n",
377 "mod = LM(params)"
378 ]
379 },
380 {
381 "cell_type": "code",
382 "execution_count": null,
383 "id": "d828c15c-4078-4967-abff-c1fd15d4696d",
384 "metadata": {},
385 "outputs": [],
386 "source": [
387 "mod.fit(rnn_dat.X_train, rnn_dat.y_train)\n",
388 "preds = mod.predict(rnn_dat.X_test)\n",
389 "print(rmse(preds2, rnn_dat.y_test.flatten()))"
390 ]
391 },
392 {
393 "cell_type": "code",
394 "execution_count": null,
395 "id": "8496a32a-8269-4d6b-953e-7f33fe626789",
396 "metadata": {},
397 "outputs": [],
398 "source": []
399 },
400 {
401 "cell_type": "code",
402 "execution_count": null,
403 "id": "75ce8bf3-6efb-4dc7-b895-def92f6ce6b4",
404 "metadata": {},
405 "outputs": [],
406 "source": []
407 },
408 {
409 "cell_type": "markdown",
410 "id": "d6e089d9-e466-45bb-80f2-15c563ae21ad",
411 "metadata": {},
412 "source": [
413 "## Class RNN "
414 ]
415 },
416 {
417 "cell_type": "code",
418 "execution_count": null,
419 "id": "3d5792a1-53e3-4099-8630-1bd5e3f52dcc",
420 "metadata": {},
421 "outputs": [],
422 "source": [
423 "from tensorflow.keras import layers,models"
424 ]
425 },
426 {
427 "cell_type": "code",
428 "execution_count": null,
429 "id": "0962428e-1124-4e1f-8500-d02b26640204",
430 "metadata": {},
431 "outputs": [],
432 "source": [
433 "import importlib\n",
434 "import moisture_rnn\n",
435 "importlib.reload(moisture_rnn)\n",
436 "from moisture_rnn import EarlyStoppingCallback, ResetStatesCallback"
437 ]
438 },
439 {
440 "cell_type": "code",
441 "execution_count": null,
442 "id": "08733f57-493a-45e7-ac07-991d92073fb6",
443 "metadata": {},
444 "outputs": [],
445 "source": [
446 "from tensorflow.keras.callbacks import Callback, EarlyStopping, TerminateOnNaN"
447 ]
448 },
449 {
450 "cell_type": "code",
451 "execution_count": null,
452 "id": "cb4d2490-9a74-4cd4-b496-2c0397e4913b",
453 "metadata": {},
454 "outputs": [],
455 "source": [
456 "class RNN2():\n",
457 " \"\"\"\n",
458 " TEST\n",
459 " \"\"\"\n",
460 " def __init__(self, params: dict):\n",
461 " \"\"\"\n",
462 " Initializes the RNNModel with the given parameters.\n",
463 "\n",
464 " Parameters:\n",
465 " -----------\n",
466 " params : dict\n",
467 " A dictionary containing model parameters.\n",
468 " \"\"\"\n",
469 " self.params = params\n",
470 " # Build model architectures based on input params\n",
471 " self.model_train = self._build_model_train()\n",
472 " self.model_predict = self._build_model_predict()\n",
473 " # Compile Models\n",
474 " optimizer=tf.keras.optimizers.Adam(learning_rate=self.params['learning_rate'])\n",
475 " self.model_train.compile(loss='mean_squared_error', optimizer=optimizer)\n",
476 " self.model_predict.compile(loss='mean_squared_error', optimizer=optimizer)\n",
477 "\n",
478 " def _build_hidden_layers(self, x, stateful):\n",
479 " params = self.params\n",
480 " \n",
481 " # Loop over each layer specified in 'hidden_layers'\n",
482 " for i, layer_type in enumerate(params['hidden_layers']):\n",
483 " units = params['hidden_units'][i]\n",
484 " activation = params['hidden_activation'][i]\n",
485 " \n",
486 " if layer_type == 'dense':\n",
487 " x = layers.Dense(units=units, activation=activation)(x)\n",
488 " \n",
489 " elif layer_type == 'dropout':\n",
490 " x = layers.Dropout(params['dropout'])(x)\n",
491 " \n",
492 " elif layer_type == 'rnn':\n",
493 " x = layers.SimpleRNN(units=units, activation=activation, dropout=params['dropout'], recurrent_dropout=params['recurrent_dropout'],\n",
494 " return_sequences=True, stateful=stateful)(x)\n",
495 " \n",
496 " elif layer_type == 'lstm':\n",
497 " x = layers.LSTM(units=units, activation=activation, dropout=params['dropout'], recurrent_dropout=params['recurrent_dropout'],\n",
498 " return_sequences=True, stateful=stateful)(x) \n",
499 " \n",
500 " elif layer_type == 'attention':\n",
501 " # Self-attention mechanism\n",
502 " x = layers.Attention()([x, x])\n",
503 " return x\n",
504 " \n",
505 " def _build_model_train(self):\n",
506 " params = self.params\n",
507 " \n",
508 " # Define the input layer with the specified batch size, timesteps, and features\n",
509 " inputs = tf.keras.Input(batch_shape=(params['batch_size'], params['timesteps'], params['n_features']))\n",
510 " x = inputs\n",
511 " # Build hidden layers\n",
512 " x = self._build_hidden_layers(x, stateful = params['stateful']) \n",
513 "\n",
514 " # Add the output layer\n",
515 " if params['output_layer'] == 'dense':\n",
516 " outputs = layers.Dense(units=params['output_dimension'], activation=params['output_activation'])(x)\n",
517 " else:\n",
518 " raise ValueError(\"Unsupported output layer type: {}\".format(params['output_layer']))\n",
519 " \n",
520 " # Create the model\n",
521 " model = models.Model(inputs=inputs, outputs=outputs)\n",
522 " return model\n",
523 "\n",
524 " def _build_model_predict(self, return_sequences=True):\n",
525 " params = self.params\n",
526 " \n",
527 " # Define the input layer with flexible batch size and sequence length\n",
528 " inputs = tf.keras.Input(shape=(None, params['n_features']))\n",
529 " x = inputs\n",
530 " # Build hidden layers\n",
531 " x = self._build_hidden_layers(x, stateful=False) \n",
532 " \n",
533 " # Add the output layer\n",
534 " if params['output_layer'] == 'dense':\n",
535 " outputs = layers.Dense(units=params['output_dimension'], activation=params['output_activation'])(x)\n",
536 " else:\n",
537 " raise ValueError(\"Unsupported output layer type: {}\".format(params['output_layer']))\n",
538 " \n",
539 " # Create the prediction model\n",
540 " model = models.Model(inputs=inputs, outputs=outputs)\n",
541 " return model\n",
542 "\n",
543 " def _setup_callbacks(self, val=False):\n",
544 "\n",
545 " callbacks = [TerminateOnNaN()]\n",
546 " \n",
547 " if self.params[\"reset_states\"]:\n",
548 " print(\"Using ResetStatesCallback.\")\n",
549 " callbacks=callbacks+[ResetStatesCallback(self.params)]\n",
550 "\n",
551 " if val:\n",
552 " print(\"Using EarlyStoppingCallback\")\n",
553 " early_stop = EarlyStoppingCallback(patience = self.params['early_stopping_patience'])\n",
554 " callbacks=callbacks+[early_stop]\n",
555 " \n",
556 " return callbacks\n",
557 "\n",
558 " def is_stateful(self):\n",
559 " \"\"\"\n",
560 " Checks whether any of the layers in the internal model (self.model_train) are stateful.\n",
561 "\n",
562 " Returns:\n",
563 " bool: True if at least one layer in the model is stateful, False otherwise.\n",
564 " \n",
565 " This method iterates over all the layers in the model and checks if any of them\n",
566 " have the 'stateful' attribute set to True. This is useful for determining if \n",
567 " the model is designed to maintain state across batches during training.\n",
568 "\n",
569 " Example:\n",
570 " --------\n",
571 " model.is_stateful()\n",
572 " \"\"\" \n",
573 " for layer in self.model_train.layers:\n",
574 " if hasattr(layer, 'stateful') and layer.stateful:\n",
575 " return True\n",
576 " return False\n",
577 "\n",
578 " def plot_history(self, history, plot_title, create_figure=True):\n",
579 " \"\"\"\n",
580 " Plots the training history. Uses log scale on y axis for readability.\n",
581 "\n",
582 " Parameters:\n",
583 " -----------\n",
584 " history : History object\n",
585 " The training history object from model fitting. Output of keras' .fit command\n",
586 " plot_title : str\n",
587 " The title for the plot.\n",
588 " \"\"\"\n",
589 " \n",
590 " if create_figure:\n",
591 " plt.figure(figsize=(10, 6))\n",
592 " plt.semilogy(history.history['loss'], label='Training loss')\n",
593 " if 'val_loss' in history.history:\n",
594 " plt.semilogy(history.history['val_loss'], label='Validation loss')\n",
595 " plt.title(f'{plot_title} Model loss')\n",
596 " plt.ylabel('Loss')\n",
597 " plt.xlabel('Epoch')\n",
598 " plt.legend(loc='upper left')\n",
599 " plt.show()\n",
600 "\n",
601 "\n",
602 " def fit(self, X_train, y_train, verbose_fit = False, verbose_weights=False, \n",
603 " plot_history=True, plot_title = '', \n",
604 " weights=None, callbacks=[], validation_data=None, return_epochs=False, *args, **kwargs):\n",
605 " \"\"\"\n",
606 " Trains the model on the provided training data. Uses the fit method of the training model and then copies the weights over to the prediction model, which has a less restrictive input shape. Formats a list of callbacks to use within the fit method based on params input\n",
607 "\n",
608 " Parameters:\n",
609 " -----------\n",
610 " X_train : np.ndarray\n",
611 " The input matrix data for training.\n",
612 " y_train : np.ndarray\n",
613 " The target vector data for training.\n",
614 " plot_history : bool, optional\n",
615 " If True, plots the training history. Default is True.\n",
616 " plot_title : str, optional\n",
617 " The title for the training plot. Default is an empty string.\n",
618 " weights : optional\n",
619 " Initial weights for the model. Default is None.\n",
620 " callbacks : list, optional\n",
621 " A list of callback functions to use during training. Default is an empty list.\n",
622 " validation_data : tuple, optional\n",
623 " Validation data to use during training, expected format (X_val, y_val). Default is None.\n",
624 " return_epochs : bool\n",
625 " If True, return the number of epochs that training took. Used to test and optimize early stopping\n",
626 " \"\"\" \n",
627 " if verbose_weights:\n",
628 " print(f\"Training simple RNN with params: {self.params}\")\n",
629 " \n",
630 " # Setup callbacks\n",
631 " # if self.params[\"reset_states\"]:\n",
632 " # callbacks=callbacks+[ResetStatesCallback(self.params), TerminateOnNaN()]\n",
633 " \n",
634 " # Check if validation data exists to modify callbacks\n",
635 " val = validation_data is not None\n",
636 " callbacks = self._setup_callbacks(val)\n",
637 " \n",
638 " # if validation_data is not None:\n",
639 " history = self.model_train.fit(\n",
640 " X_train, y_train, \n",
641 " epochs=self.params['epochs'], \n",
642 " batch_size=self.params['batch_size'],\n",
643 " callbacks = callbacks,\n",
644 " verbose=verbose_fit,\n",
645 " validation_data = validation_data,\n",
646 " *args, **kwargs\n",
647 " )\n",
648 " \n",
649 " if plot_history:\n",
650 " self.plot_history(history,plot_title)\n",
651 " \n",
652 " if verbose_weights:\n",
653 " print(f\"Fitted Weights Hash: {hash_weights(self.model_train)}\")\n",
654 "\n",
655 " # Update Weights for Prediction Model\n",
656 " w_fitted = self.model_train.get_weights()\n",
657 " self.model_predict.set_weights(w_fitted)\n",
658 "\n",
659 " if return_epochs:\n",
660 " # Epoch counting starts at 0, adding 1 for the count\n",
661 " return early_stop.best_epoch + 1\n",
662 "\n",
663 " def predict(self, X_test, verbose=True):\n",
664 " if verbose:\n",
665 " print(\"Predicting test data\")\n",
666 " preds = self.model_predict.predict(X_test)\n",
667 " \n",
668 " return preds\n",
669 "\n",
670 " def run_model(self, data, reproducibility_run=False, plot_period='all', return_epochs=False):\n",
671 "\n",
672 " # Set up print statements with verbose args\n",
673 " verbose_fit = self.params['verbose_fit']\n",
674 " verbose_weights = self.params['verbose_weights']\n",
675 " if verbose_weights:\n",
676 " data.print_hashes() \n",
677 "\n",
678 " # Set up name for run, used for plotting\n",
679 " case_id = \"Spatial Training Set\" if data.spatial else data.id\n",
680 " print(f\"Running {case_id}\")\n",
681 "\n",
682 " # Extract Datasets\n",
683 " X_train, y_train, X_test, y_test = data.X_train, data.y_train, data.X_test, data.y_test\n",
684 " X_val, y_val = data.get('X_val', None), data.get('y_val', None)\n",
685 " validation_data = (X_val, y_val) if X_val is not None and y_val is not None else None\n",
686 " \n",
687 " # Fit model, assign epochs to object, will just asign None if return_epochs is false\n",
688 " # NOTE: when using early stopping, number of epochs much be extracted here at the fit call\n",
689 " eps = self.fit(X_train, y_train, validation_data=validation_data, plot_title=case_id, return_epochs=return_epochs, verbose_fit=verbose_fit)\n",
690 "\n",
691 " # Generate Predictions\n",
692 " m = self.predict(X_test)\n",
693 " errs = eval_errs(m, y_test)\n",
694 "\n",
695 " return m, errs\n",
696 "\n",
697 "\n",
698 "def eval_errs(preds, y_test):\n",
699 " \"\"\"\n",
700 " Calculate RMSE for ndarrays structured as (batch_size, timesteps, features). \n",
701 " The first dimension, batch_size, could denote distinct locations. The second, timesteps, is length of sequence\n",
702 " \"\"\"\n",
703 " squared_diff = np.square(preds - y_test)\n",
704 " \n",
705 " # Mean squared error along the timesteps and dimensions (axis 1 and 2)\n",
706 " mse = np.mean(squared_diff, axis=(1, 2))\n",
707 " \n",
708 " # Root mean squared error (RMSE) for each timeseries\n",
709 " rmses = np.mean(np.sqrt(mse))\n",
710 " \n",
711 " return rmses"
712 ]
713 },
714 {
715 "cell_type": "code",
716 "execution_count": null,
717 "id": "e31ca700-7871-4fb5-b5d2-9b63b85f6688",
718 "metadata": {},
719 "outputs": [],
720 "source": []
721 },
722 {
723 "cell_type": "code",
724 "execution_count": null,
725 "id": "42459583-a634-4dd9-a94b-4535302f481d",
726 "metadata": {},
727 "outputs": [],
728 "source": [
729 "params_test = {\n",
730 " 'n_features': 8,\n",
731 " 'timesteps': 12,\n",
732 " 'batch_size': 32,\n",
733 " 'hidden_layers': ['dense', 'lstm', 'attention', 'dense', 'dense'],\n",
734 " 'hidden_units': [64, 32, None, 32, 16],\n",
735 " 'hidden_activation': ['relu', 'tanh', None, 'relu', 'relu'],\n",
736 " 'dropout': 0.2,\n",
737 " 'recurrent_dropout': 0.2,\n",
738 " 'output_layer': 'dense',\n",
739 " 'output_activation': 'linear',\n",
740 " 'output_dimension': 1,\n",
741 " 'learning_rate': 0.001,\n",
742 " 'early_stopping_patience': 5,\n",
743 " 'epochs': 3,\n",
744 " 'reset_states': True,\n",
745 " 'bmin': 10,\n",
746 " 'bmax': 200,\n",
747 " 'batch_schedule_type': 'step',\n",
748 " 'estep': 5,\n",
749 " 'features_list': ['Ed', 'Ew', 'rain', 'elev', 'lon', 'lat', 'solar', 'wind'],\n",
750 " 'scaler': 'standard',\n",
751 " 'time_fracs': [.8, .1, .1],\n",
752 " 'space_fracs': [.8, .1, .1],\n",
753 " 'stateful': True,\n",
754 " 'verbose_fit': True,\n",
755 " 'verbose_weights': False,\n",
756 " # 'return_sequences': True # whether or not the LAST recurrent layer should return sequences. If multiple, all previous need to be True\n",
757 "}"
758 ]
759 },
760 {
761 "cell_type": "code",
762 "execution_count": null,
763 "id": "d7e7dc43-021c-4152-b1c5-dcf3ae25b5b1",
764 "metadata": {},
765 "outputs": [],
766 "source": [
767 "params_test"
768 ]
769 },
770 {
771 "cell_type": "code",
772 "execution_count": null,
773 "id": "aa9117f1-39be-4ce0-81bc-c3bbbdf0aa95",
774 "metadata": {},
775 "outputs": [],
776 "source": [
777 "params_test['scaler']"
778 ]
779 },
780 {
781 "cell_type": "code",
782 "execution_count": null,
783 "id": "7c1627f9-f011-4159-98a2-1b5973929e71",
784 "metadata": {},
785 "outputs": [],
786 "source": [
787 "reproducibility.set_seed()\n",
788 "mod = RNN2(params_test)"
789 ]
790 },
791 {
792 "cell_type": "code",
793 "execution_count": null,
794 "id": "5dbc66c0-ccb5-46c2-a073-1fa7a5be750a",
795 "metadata": {},
796 "outputs": [],
797 "source": [
798 "mod.model_train.summary()"
799 ]
800 },
801 {
802 "cell_type": "code",
803 "execution_count": null,
804 "id": "30498201-3798-484d-922f-974909b195af",
805 "metadata": {},
806 "outputs": [],
807 "source": [
808 "rnn_dat = rnn_data_wrap(combine_nested(train3), params_test)"
809 ]
810 },
811 {
812 "cell_type": "code",
813 "execution_count": null,
814 "id": "e213ffd7-d26c-41ce-8e2b-b17368fdd7a8",
815 "metadata": {},
816 "outputs": [],
817 "source": [
818 "params.update({\n",
819 " 'loc_batch_reset': rnn_dat.n_seqs # Used to reset hidden state when location changes for a given batch\n",
820 "})"
821 ]
822 },
823 {
824 "cell_type": "code",
825 "execution_count": null,
826 "id": "eb6f246f-c02e-434e-89d1-f6d9ec1c6b26",
827 "metadata": {},
828 "outputs": [],
829 "source": [
830 "reproducibility.set_seed()\n",
831 "mod3 = RNN3(params_test)\n",
832 "mod3.model_train.summary()"
833 ]
834 },
835 {
836 "cell_type": "code",
837 "execution_count": null,
838 "id": "74e599b6-7f4d-4175-a5f1-de892e72ebd4",
839 "metadata": {},
840 "outputs": [],
841 "source": []
842 },
843 {
844 "cell_type": "code",
845 "execution_count": null,
846 "id": "f894d203-d277-48f3-bb57-a610f162361f",
847 "metadata": {},
848 "outputs": [],
849 "source": []
850 },
851 {
852 "cell_type": "code",
853 "execution_count": null,
854 "id": "6a3f38aa-ed3f-4511-a7ec-f3403aa4c717",
855 "metadata": {},
856 "outputs": [],
857 "source": []
858 },
859 {
860 "cell_type": "code",
861 "execution_count": null,
862 "id": "66ddc2cd-0308-4622-bbb1-d26d08292159",
863 "metadata": {},
864 "outputs": [],
865 "source": []
866 },
867 {
868 "cell_type": "markdown",
869 "id": "282cb651-b21f-401d-94c5-9e07530a9ba8",
870 "metadata": {},
871 "source": [
872 "## RNN"
873 ]
874 },
875 {
876 "cell_type": "code",
877 "execution_count": null,
878 "id": "8c1894e3-5283-4e5e-83ae-9c386836a990",
879 "metadata": {},
880 "outputs": [],
881 "source": [
882 "import importlib \n",
883 "import moisture_rnn\n",
884 "importlib.reload(moisture_rnn)\n",
885 "from moisture_rnn import RNN"
886 ]
887 },
888 {
889 "cell_type": "code",
890 "execution_count": null,
891 "id": "aa1b690f-edaa-4c97-893c-ec9a3a615ce1",
892 "metadata": {},
893 "outputs": [],
894 "source": [
895 "params = read_yml(\"params.yaml\", subkey=\"lstm\")\n",
896 "params = RNNParams(params)\n",
897 "params.update({\n",
898 " 'dense_layers': 2,\n",
899 " 'dense_units': 32\n",
900 "})"
901 ]
902 },
903 {
904 "cell_type": "code",
905 "execution_count": null,
906 "id": "054ab015-4e41-4255-8b1a-843b61e3d21d",
907 "metadata": {},
908 "outputs": [],
909 "source": [
910 "params.update({'batch_schedule_type': 'step'})"
911 ]
912 },
913 {
914 "cell_type": "code",
915 "execution_count": null,
916 "id": "fa38f35a-d367-4df8-b2d3-7691ff4b0cf4",
917 "metadata": {},
918 "outputs": [],
919 "source": [
920 "rnn_dat = rnn_data_wrap(combine_nested(train3), params)\n",
921 "reproducibility.set_seed(123)\n",
922 "rnn = RNN(params)"
923 ]
924 },
925 {
926 "cell_type": "code",
927 "execution_count": null,
928 "id": "ade176b9-2844-43b6-b85e-5bb30414aa35",
929 "metadata": {},
930 "outputs": [],
931 "source": [
932 "rnn.params"
933 ]
934 },
935 {
936 "cell_type": "code",
937 "execution_count": null,
938 "id": "5945e6c1-6b3a-4b7d-ade2-b5788860ef18",
939 "metadata": {},
940 "outputs": [],
941 "source": [
942 "rnn.model_train.fit(rnn_dat.X_train, rnn_dat.y_train, validation_data=(rnn_dat.X_val, rnn_dat.y_val), \n",
943 " verbose=True, epochs=20)"
944 ]
945 },
946 {
947 "cell_type": "code",
948 "execution_count": null,
949 "id": "2d123b2b-047e-4a04-b49e-6629cc22edc6",
950 "metadata": {},
951 "outputs": [],
952 "source": [
953 "rnn.model_predict.set_weights(rnn.model_train.get_weights())"
954 ]
955 },
956 {
957 "cell_type": "code",
958 "execution_count": null,
959 "id": "db57df64-d2ac-4b91-bbfc-71a5834ddf41",
960 "metadata": {},
961 "outputs": [],
962 "source": [
963 "rnn.model_predict.summary()"
964 ]
965 },
966 {
967 "cell_type": "code",
968 "execution_count": null,
969 "id": "0466887f-9833-4a6a-a0c7-a4d56f207d33",
970 "metadata": {},
971 "outputs": [],
972 "source": [
973 "rnn_dat.X_test.shape"
974 ]
975 },
976 {
977 "cell_type": "code",
978 "execution_count": null,
979 "id": "1d3e630c-db69-4603-962e-95c576b45ac9",
980 "metadata": {},
981 "outputs": [],
982 "source": [
983 "preds = rnn.model_predict.predict(rnn_dat.X_test)"
984 ]
985 },
986 {
987 "cell_type": "code",
988 "execution_count": null,
989 "id": "8b8228a9-5b6d-4de1-8968-d40277edacd2",
990 "metadata": {},
991 "outputs": [],
992 "source": [
993 "preds.shape"
994 ]
995 },
996 {
997 "cell_type": "code",
998 "execution_count": null,
999 "id": "8b001dd8-ffd7-4fd1-bf11-413515ddc488",
1000 "metadata": {},
1001 "outputs": [],
1002 "source": [
1003 "rnn_dat.X_test.shape"
1004 ]
1005 },
1006 {
1007 "cell_type": "code",
1008 "execution_count": null,
1009 "id": "f96c6dbf-6ca8-451e-abc4-b68b8116871b",
1010 "metadata": {},
1011 "outputs": [],
1012 "source": [
1013 "squared_diff = np.square(preds - rnn_dat.y_test)\n",
1014 "mse = np.mean(squared_diff, axis=(1, 2))\n",
1015 "errs = np.sqrt(mse)\n",
1016 "errs.mean()"
1017 ]
1018 },
1019 {
1020 "cell_type": "markdown",
1021 "id": "5ef092ff-8af1-491a-b0bf-cc3e674330e0",
1022 "metadata": {},
1023 "source": [
1024 "## Phys Initialized"
1025 ]
1026 },
1027 {
1028 "cell_type": "code",
1029 "execution_count": null,
1030 "id": "5488628e-4552-4909-83e9-413fd6878bdd",
1031 "metadata": {},
1032 "outputs": [],
1033 "source": [
1034 "params.update({\n",
1035 " 'epochs':100,\n",
1036 " 'dense_layers': 0,\n",
1037 " 'activation': ['relu', 'relu'],\n",
1038 " 'phys_initialize': False,\n",
1039 " 'dropout': [0,0],\n",
1040 " 'space_fracs': [.8, .1, .1],\n",
1041 " 'scaler': None\n",
1042 "})"
1043 ]
1044 },
1045 {
1046 "cell_type": "code",
1047 "execution_count": null,
1048 "id": "ab7db7d6-949e-457d-90b9-22d9c5aa4739",
1049 "metadata": {},
1050 "outputs": [],
1051 "source": [
1052 "import importlib\n",
1053 "import moisture_rnn\n",
1054 "importlib.reload(moisture_rnn)\n",
1055 "from moisture_rnn import rnn_data_wrap"
1056 ]
1057 },
1058 {
1059 "cell_type": "code",
1060 "execution_count": null,
1061 "id": "d26cf1b2-2fad-409d-888f-4921b0ae4ba8",
1062 "metadata": {},
1063 "outputs": [],
1064 "source": [
1065 "params['scaler'] is None"
1066 ]
1067 },
1068 {
1069 "cell_type": "code",
1070 "execution_count": null,
1071 "id": "1c4627bc-0f90-44e6-9103-2efe5c5f439d",
1072 "metadata": {},
1073 "outputs": [],
1074 "source": [
1075 "rnn_dat = rnn_data_wrap(combine_nested(train3), params)"
1076 ]
1077 },
1078 {
1079 "cell_type": "code",
1080 "execution_count": null,
1081 "id": "56bdf26c-07e7-4e4a-a567-af7dd0f564d9",
1082 "metadata": {},
1083 "outputs": [],
1084 "source": [
1085 "reproducibility.set_seed()\n",
1086 "rnn = RNN(params)\n",
1087 "m, errs = rnn.run_model(rnn_dat)"
1088 ]
1089 },
1090 {
1091 "cell_type": "code",
1092 "execution_count": null,
1093 "id": "01227b79-98f3-4931-bdfc-ff08afa8be5f",
1094 "metadata": {},
1095 "outputs": [],
1096 "source": [
1097 "rnn.model_train.summary()"
1098 ]
1099 },
1100 {
1101 "cell_type": "code",
1102 "execution_count": null,
1103 "id": "918a8bf0-638b-4b4b-82fe-c6a1965a72dd",
1104 "metadata": {},
1105 "outputs": [],
1106 "source": [
1107 "errs.mean()"
1108 ]
1109 },
1110 {
1111 "cell_type": "code",
1112 "execution_count": null,
1113 "id": "37fdbb3a-3e83-4541-93b2-982b6d4cbe93",
1114 "metadata": {
1115 "scrolled": true
1116 },
1117 "outputs": [],
1118 "source": [
1119 "rnn_dat.X_train[:,:,0].mean()"
1120 ]
1121 },
1122 {
1123 "cell_type": "code",
1124 "execution_count": null,
1125 "id": "7ca41db1-72aa-44b6-b9dd-058735336ab3",
1126 "metadata": {},
1127 "outputs": [],
1128 "source": []
1129 },
1130 {
1131 "cell_type": "code",
1132 "execution_count": null,
1133 "id": "a592a4c9-cb3b-4174-8eaa-02afd00a1897",
1134 "metadata": {},
1135 "outputs": [],
1136 "source": [
1137 "rnn_dat['features_list']"
1138 ]
1139 },
1140 {
1141 "cell_type": "code",
1142 "execution_count": null,
1143 "id": "3832fb05-417c-4648-8e2e-7748c06b3768",
1144 "metadata": {},
1145 "outputs": [],
1146 "source": []
1147 },
1148 {
1149 "cell_type": "markdown",
1150 "id": "d2360aef-e9c4-4a71-922d-336e53b82537",
1151 "metadata": {},
1152 "source": [
1153 "## LSTM"
1154 ]
1155 },
1156 {
1157 "cell_type": "code",
1158 "execution_count": null,
1159 "id": "71d4e441-9bf1-4d57-bb37-091553e23212",
1160 "metadata": {},
1161 "outputs": [],
1162 "source": [
1163 "import importlib \n",
1164 "import moisture_rnn\n",
1165 "importlib.reload(moisture_rnn)\n",
1166 "from moisture_rnn import RNN_LSTM"
1167 ]
1168 },
1169 {
1170 "cell_type": "code",
1171 "execution_count": null,
1172 "id": "0f6ba896-e3be-4a9f-8a42-3df64aff7d63",
1173 "metadata": {},
1174 "outputs": [],
1175 "source": [
1176 "params = read_yml(\"params.yaml\", subkey=\"lstm\")\n",
1177 "params = RNNParams(params)"
1178 ]
1179 },
1180 {
1181 "cell_type": "code",
1182 "execution_count": null,
1183 "id": "a4cf567e-d623-4e14-b578-eed88b80d04e",
1184 "metadata": {},
1185 "outputs": [],
1186 "source": [
1187 "rnn_dat = rnn_data_wrap(combine_nested(train3), params)"
1188 ]
1189 },
1190 {
1191 "cell_type": "code",
1192 "execution_count": null,
1193 "id": "57bb5708-7be9-4474-abb4-3b7ff4bf79df",
1194 "metadata": {},
1195 "outputs": [],
1196 "source": [
1197 "params.update({\n",
1198 " 'loc_batch_reset': rnn_dat.n_seqs # Used to reset hidden state when location changes for a given batch\n",
1199 "})"
1200 ]
1201 },
1202 {
1203 "cell_type": "code",
1204 "execution_count": null,
1205 "id": "0157a6bc-3a99-4b87-a42c-ab770d19ae37",
1206 "metadata": {},
1207 "outputs": [],
1208 "source": [
1209 "from moisture_rnn import ResetStatesCallback, EarlyStoppingCallback\n",
1210 "params.update({'epochs': 50, 'learning_rate': 0.001, 'verbose_fit': True, 'rnn_layers': 2, 'rnn_units': 20, 'dense_layers': 1, 'dense_units': 10,\n",
1211 " 'activation': ['tanh', 'tanh'], 'features_list': rnn_dat.features_list,\n",
1212 " 'batch_schedule_type':'step', 'bmin': 10, 'bmax':rnn_dat.hours})\n",
1213 "reproducibility.set_seed(123)\n",
1214 "lstm = RNN_LSTM(params)\n",
1215 "\n",
1216 "history = lstm.model_train.fit(rnn_dat.X_train, rnn_dat.y_train, \n",
1217 " batch_size = params['batch_size'], epochs=params['epochs'], \n",
1218 " callbacks = [ResetStatesCallback(params),\n",
1219 " EarlyStoppingCallback(patience = 15)],\n",
1220 " validation_data = (rnn_dat.X_val, rnn_dat.y_val))\n",
1221 " "
1222 ]
1223 },
1224 {
1225 "cell_type": "code",
1226 "execution_count": null,
1227 "id": "9b3c8d8d-ea50-44ea-8c0c-414e07cd01ac",
1228 "metadata": {},
1229 "outputs": [],
1230 "source": []
1231 },
1232 {
1233 "cell_type": "code",
1234 "execution_count": null,
1235 "id": "03063e3c-e8f4-451d-b0cf-25bd965cd9d6",
1236 "metadata": {},
1237 "outputs": [],
1238 "source": [
1239 "params = RNNParams(read_yml(\"params.yaml\", subkey=\"lstm\"))\n",
1240 "params.update({'epochs': 50, 'learning_rate': 0.001, 'verbose_fit': True, 'rnn_layers': 2, 'rnn_units': 20, 'dense_layers': 1, 'dense_units': 10,\n",
1241 " 'activation': ['tanh', 'tanh'], 'features_list': rnn_dat.features_list,\n",
1242 " 'batch_schedule_type':'step', 'bmin': 10, 'bmax':rnn_dat.hours})\n",
1243 "rnn_dat = rnn_data_wrap(combine_nested(train3), params)\n",
1244 "params.update({\n",
1245 " 'loc_batch_reset': rnn_dat.n_seqs # Used to reset hidden state when location changes for a given batch\n",
1246 "})\n",
1247 "reproducibility.set_seed(123)\n",
1248 "lstm = RNN_LSTM(params)\n",
1249 "m, errs = lstm.run_model(rnn_dat)"
1250 ]
1251 },
1252 {
1253 "cell_type": "code",
1254 "execution_count": null,
1255 "id": "be46a2dc-bf5c-4893-a1ee-a1682566f7a2",
1256 "metadata": {},
1257 "outputs": [],
1258 "source": [
1259 "errs.mean()"
1260 ]
1261 },
1262 {
1263 "cell_type": "code",
1264 "execution_count": null,
1265 "id": "0f319f37-7d13-41fd-95fa-66dbdfeab588",
1266 "metadata": {},
1267 "outputs": [],
1268 "source": []
1269 },
1270 {
1271 "cell_type": "code",
1272 "execution_count": null,
1273 "id": "b1252b08-62b9-4d24-add2-0f87d15b0ff2",
1274 "metadata": {},
1275 "outputs": [],
1276 "source": [
1277 "params = RNNParams(read_yml(\"params.yaml\", subkey=\"rnn\"))\n",
1278 "rnn_dat = rnn_data_wrap(combine_nested(train3), params)"
1279 ]
1280 },
1281 {
1282 "cell_type": "code",
1283 "execution_count": null,
1284 "id": "9281540b-eb26-4923-883b-1b31d8347634",
1285 "metadata": {},
1286 "outputs": [],
1287 "source": [
1288 "reproducibility.set_seed(123)\n",
1289 "rnn = RNN(params)\n",
1290 "m, errs = rnn.run_model(rnn_dat)"
1291 ]
1292 },
1293 {
1294 "cell_type": "code",
1295 "execution_count": null,
1296 "id": "8a0269b4-d6b7-4f20-8386-69814d7acaa3",
1297 "metadata": {},
1298 "outputs": [],
1299 "source": [
1300 "errs.mean()"
1301 ]
1302 },
1303 {
1304 "cell_type": "code",
1305 "execution_count": null,
1306 "id": "10b44de3-a0e9-49e4-9e03-873d69580c07",
1307 "metadata": {},
1308 "outputs": [],
1309 "source": []
1310 },
1311 {
1312 "cell_type": "code",
1313 "execution_count": null,
1314 "id": "27f4fee4-7fce-49c5-a455-97a90b754c13",
1315 "metadata": {},
1316 "outputs": [],
1317 "source": []
1318 },
1319 {
1320 "cell_type": "code",
1321 "execution_count": null,
1322 "id": "739d4b26-641e-47b2-a90a-67cd32215d05",
1323 "metadata": {},
1324 "outputs": [],
1325 "source": []
1326 }
1327 ],
1328 "metadata": {
1329 "kernelspec": {
1330 "display_name": "Python 3 (ipykernel)",
1331 "language": "python",
1332 "name": "python3"
1333 },
1334 "language_info": {
1335 "codemirror_mode": {
1336 "name": "ipython",
1337 "version": 3
1338 },
1339 "file_extension": ".py",
1340 "mimetype": "text/x-python",
1341 "name": "python",
1342 "nbconvert_exporter": "python",
1343 "pygments_lexer": "ipython3",
1344 "version": "3.12.5"
1345 }
1346 },
1347 "nbformat": 4,
1348 "nbformat_minor": 5
1349 }