4 "cell_type": "markdown",
5 "id": "244c2fb0-4339-476c-a2db-a641e124e25a",
8 "# v2.1 exploration trying to make it work better"
13 "execution_count": null,
14 "id": "e6cc7920-e380-4b81-bac0-cd6840450e9a",
20 "import os.path as osp\n",
21 "import numpy as np\n",
22 "import pandas as pd\n",
23 "import tensorflow as tf\n",
24 "import matplotlib.pyplot as plt\n",
27 "sys.path.append('..')\n",
28 "import reproducibility\n",
29 "import pandas as pd\n",
30 "from utils import print_dict_summary\n",
31 "from data_funcs import rmse\n",
32 "from moisture_rnn import RNNParams, RNNData, RNN, RNN_LSTM, create_rnn_data2\n",
33 "from moisture_rnn_pkl import pkl2train\n",
34 "from tensorflow.keras.callbacks import Callback\n",
35 "from utils import hash2\n",
39 "from utils import logging_setup, read_yml, read_pkl, hash_ndarray, hash_weights\n",
46 "execution_count": null,
47 "id": "f58e8839-bf0e-4995-b966-c09e4df001ce",
55 "cell_type": "markdown",
56 "id": "b62f4360-e9d1-4510-bb5d-1d79a3a5ac75",
64 "execution_count": null,
65 "id": "91155de8-1927-4485-80c9-095fb4d8613d",
71 "# data_file = 'data/test_CA_202401.pkl'\n",
72 "# train = pkl2train([data_file])\n",
73 "# # with open(\"train.pkl\", 'wb') as file:\n",
74 "# # logging.info('Writing the rain cases into file %s',\"train.pkl\")\n",
75 "# # pickle.dump(train, file)\n",
78 "train = read_pkl(\"train.pkl\")"
83 "execution_count": null,
84 "id": "a5979f46-7e1a-4552-a92f-fc17d2ca75b4",
93 "execution_count": null,
94 "id": "1bc6a3c7-e068-4d83-9ab0-131eaeec289e",
98 "case = \"FBFW1_202401\"\n",
104 "execution_count": null,
105 "id": "84a4e8b6-9762-4406-b71e-bb6bf990fd9b",
109 "params = read_yml(\"params.yaml\", subkey=\"rnn\")\n",
110 "params = RNNParams(params)\n",
116 "execution_count": null,
117 "id": "92b7a5ed-5215-43a7-933c-5dd146241944",
121 "rnn_dat = RNNData(dat, scaler = params['scaler'], features_list = params['features_list'])\n",
122 "rnn_dat.train_test_split(\n",
123 " train_frac = .9,\n",
126 "rnn_dat.scale_data()"
131 "execution_count": null,
132 "id": "9141a034-5912-4087-aaaa-a4c221e77495",
136 "import importlib\n",
137 "import moisture_rnn\n",
138 "importlib.reload(moisture_rnn)\n",
139 "from moisture_rnn import RNN"
144 "execution_count": null,
145 "id": "2c570a67-0b67-4fe4-a536-68cf5a1f8256",
149 "params.update({'epochs': 20})\n",
150 "reproducibility.set_seed(123)\n",
151 "rnn = RNN(params)\n"
156 "execution_count": null,
157 "id": "034300ff-a71e-4807-85d7-368197962ab5",
161 "m, errs = rnn.run_model(rnn_dat)"
166 "execution_count": null,
167 "id": "1f5183e0-5e58-4ff3-9a1f-0f707241a698",
176 "execution_count": null,
177 "id": "0dc33ddc-cb10-4b0c-b7f2-af3302e28a23",
184 "execution_count": null,
185 "id": "ebb4d0d3-0a9e-428e-bd56-6899bde36df7",
191 "cell_type": "markdown",
192 "id": "a42cc05f-1438-459f-9a15-64276aa2f651",
200 "execution_count": null,
201 "id": "d27c7277-c9e1-4fd1-b050-d4b6bc737822",
205 "import importlib\n",
207 "importlib.reload(utils)\n",
208 "from utils import read_pkl"
213 "execution_count": null,
214 "id": "7a9a5414-d33f-4f0d-b1e5-b320b22d60f5",
218 "train = read_pkl(\"train.pkl\")"
223 "execution_count": null,
224 "id": "e0147836-f6ba-4141-9c9d-7c2e5d676bc2",
228 "params = read_yml(\"params.yaml\", subkey=\"rnn\")\n",
229 "params = RNNParams(params)"
234 "execution_count": null,
235 "id": "4219f8e0-cf44-43a6-830b-fc859c3d954b",
239 "params.update({'activation': ['linear', 'linear'], 'epochs':300, 'val_frac': .2, 'scaler': 'minmax', 'rnn_layers': 1, 'dense_layers': 1})"
244 "execution_count": null,
245 "id": "722bd100-beaa-49c7-a1ab-b72765c89ebe",
249 "rnn_dat = RNNData(train['NV020_202401'], scaler = params['scaler'], features_list = params['features_list'])"
254 "execution_count": null,
255 "id": "2eb4adf9-c4eb-493c-9f62-59ba17f6da2f",
259 "rnn_dat.train_test_split(\n",
260 " train_frac = params['train_frac'],\n",
261 " val_frac = params['val_frac']\n",
267 "execution_count": null,
268 "id": "2b4fa4e1-c1b9-483a-83ac-cf0ee46662fa",
272 "rnn_dat.scale_data()"
277 "execution_count": null,
278 "id": "fc362aa4-fe28-4848-9575-bb17f72ac9fd",
282 "import importlib\n",
283 "import moisture_rnn\n",
284 "importlib.reload(moisture_rnn)\n",
285 "from moisture_rnn import RNN"
290 "execution_count": null,
291 "id": "79381a0b-2338-4a09-876e-91e50b968d3f",
300 "execution_count": null,
301 "id": "155cebc4-e7f6-47ae-943e-556c3939ab95",
305 "rnn.predict(rnn_dat.X_test)"
310 "execution_count": null,
311 "id": "7d28dd8c-3d90-43ba-842e-4d4ed5deb823",
315 "rnn.model_predict.summary()"
320 "execution_count": null,
321 "id": "1f928e06-867e-4cc5-ab94-83b30b923374",
325 "reproducibility.set_seed(123)\n",
326 "rnn = RNN(params)\n",
327 "m, errs = rnn.run_model(rnn_dat)"
332 "execution_count": null,
333 "id": "4b4ca16e-7c40-4bb9-b971-dd0efa4e8a83",
340 "execution_count": null,
341 "id": "54c917e9-20a6-4b8d-b6ab-04fdb0333467",
348 "execution_count": null,
349 "id": "7c659050-e74e-4f07-b95a-3f7b57653061",
356 "execution_count": null,
357 "id": "e86c9e4d-4ccd-4d9d-92e1-2e4299549fa4",
364 "execution_count": null,
365 "id": "966c3559-740d-44d3-b98d-cc2efe63afcd",
372 "execution_count": null,
373 "id": "52c1df9f-87ab-4882-aca1-e90cca7bd470",
380 "execution_count": null,
381 "id": "1addf015-b2c1-42df-a769-39df81dd5d14",
388 "execution_count": null,
389 "id": "3262a73f-0bd7-4e78-ade0-ca53a7da2b84",
396 "execution_count": null,
397 "id": "eecfd38b-0a1a-4de3-b568-2f53ffbcc78c",
403 "cell_type": "markdown",
404 "id": "2298a1a1-b72c-4c7e-bcb6-2cdefe96fe3e",
407 "## Test Data Creation"
412 "execution_count": null,
413 "id": "c4645246-edce-4544-9809-5ffb0760ae25",
417 "import importlib\n",
418 "import moisture_rnn_pkl\n",
419 "importlib.reload(moisture_rnn_pkl)\n",
420 "from moisture_rnn_pkl import pkl2train"
425 "execution_count": null,
426 "id": "5b662edb-7a79-4532-b0d7-2492b1ad917d",
430 "file_names=['test_CA_202401.pkl', 'test_NW_202401.pkl']\n",
432 "file_paths = [osp.join(file_dir,file_name) for file_name in file_names]"
437 "execution_count": null,
438 "id": "1185c995-e9fa-4586-96c2-44b159ccf477",
444 "train = pkl2train(file_paths)"
449 "execution_count": null,
450 "id": "665291be-0f40-46b5-9a63-27a58965f8ca",
459 "execution_count": null,
460 "id": "6b61a406-eed8-4595-9c3f-4c11e1aed7c8",
467 "execution_count": null,
468 "id": "e234b0f6-3cc9-46d1-926a-d825c58e3991",
475 "execution_count": null,
476 "id": "7fdf595c-68e1-4e93-a5ec-d6e20e2f1bdf",
483 "execution_count": null,
484 "id": "fc3e8264-da29-4261-a560-ef457f42ed70",
491 "execution_count": null,
492 "id": "7deda359-1e7f-447a-97b7-576b98712a74",
499 "execution_count": null,
500 "id": "7fc05c26-9a54-4863-8956-d76913128701",
506 "cell_type": "markdown",
507 "id": "2afc2cf7-eab1-4a85-8632-4d306aead358",
515 "execution_count": null,
516 "id": "bfd419f0-9092-470d-81b7-d3b45e4bdc0b",
523 "execution_count": null,
524 "id": "545ece65-9f4a-4b45-b87f-ea3a23032cac",
531 "execution_count": null,
532 "id": "1e9ec6f9-8598-4560-b71e-222f5b4c4968",
539 "execution_count": null,
540 "id": "e2a7840d-f7e4-424d-b343-06f913f9d3f6",
547 "execution_count": null,
548 "id": "52e2942b-3bed-4c3d-8082-c7069d791036",
555 "execution_count": null,
556 "id": "def73f2c-5d2f-42c6-8c2d-328ac5e8db20",
563 "execution_count": null,
564 "id": "888dd72a-4eef-414b-ac33-f6f4bfbefe60",
573 "execution_count": null,
574 "id": "7f40cdfd-b33a-43c1-8bc4-44a0ea6817ff",
578 "import importlib \n",
579 "import moisture_rnn\n",
580 "importlib.reload(moisture_rnn)\n",
581 "from moisture_rnn import RNN"
586 "execution_count": null,
587 "id": "bdf0ba2e-f944-4c86-a20e-a59e023897cb",
591 "params = read_yml(\"params.yaml\", subkey=\"rnn\")\n",
592 "params = RNNParams(params)"
597 "execution_count": null,
598 "id": "9dbd51b0-9342-4b90-a250-0ac2c75d3066",
602 "reproducibility.set_seed()\n",
603 "rnn = RNN(params)\n",
604 "m, errs = rnn.run_model(rnn_dat)"
609 "execution_count": null,
610 "id": "c6d7d34c-dfae-4370-a398-a287790eff53",
616 "cell_type": "markdown",
617 "id": "d2360aef-e9c4-4a71-922d-336e53b82537",
627 "execution_count": null,
628 "id": "71d4e441-9bf1-4d57-bb37-091553e23212",
632 "import importlib \n",
633 "import moisture_rnn\n",
634 "importlib.reload(moisture_rnn)\n",
635 "from moisture_rnn import RNN_LSTM"
640 "execution_count": null,
641 "id": "59480f19-3567-4b24-b6ff-d9292dc8c2ec",
645 "with open(\"params.yaml\") as file:\n",
646 " params = yaml.safe_load(file)[\"lstm\"]\n",
648 "rnn_dat2 = create_rnn_data2(train[case],params)"
653 "execution_count": null,
654 "id": "2adff592-7aa4-4e59-a229-cad4a133297e",
658 "params.update({'epochs': 10})"
663 "execution_count": null,
664 "id": "b20539f0-eed2-44de-9269-ae8696c8e7c8",
671 "execution_count": null,
672 "id": "6bfbcbb5-b631-4594-9ae5-618c4fe68e7b",
676 "reproducibility.set_seed()\n",
677 "rnn = RNN(params)\n",
678 "m, errs = rnn.run_model(rnn_dat2)"
683 "execution_count": null,
684 "id": "dd8a9700-f479-4c11-8655-ca7b45222402",
691 "execution_count": null,
692 "id": "de46c481-74a7-46cc-8334-678ad8230cce",
696 "import importlib\n",
697 "importlib.reload(moisture_rnn)\n",
698 "from moisture_rnn import RNN_LSTM"
703 "execution_count": null,
704 "id": "2b6a699a-68e8-49ef-95f2-409137502fb6",
708 "with open(\"params.yaml\") as file:\n",
709 " params = yaml.safe_load(file)[\"lstm\"]\n",
711 "rnn_dat2 = create_rnn_data2(train[case],params)\n",
717 "execution_count": null,
718 "id": "188c0d5d-f3f6-4a61-83b0-b21dfc5d01b7",
723 " 'learning_rate': 0.000001,\n",
725 " 'clipvalue':1.0\n",
731 "execution_count": null,
732 "id": "6a9d612e-8cd2-40ca-a789-91c99c3d6ccd",
736 "reproducibility.set_seed()\n",
737 "lstm = RNN_LSTM(params)\n",
738 "m, errs = lstm.run_model(rnn_dat2)"
743 "execution_count": null,
744 "id": "ec95e7d4-6d57-441b-b673-f10625ee5dec",
751 "execution_count": null,
752 "id": "9b3c8d8d-ea50-44ea-8c0c-414e07cd01ac",
759 "execution_count": null,
760 "id": "03063e3c-e8f4-451d-b0cf-25bd965cd9d6",
767 "execution_count": null,
768 "id": "f60a24c6-9a67-45aa-bc5c-8818aa0ca049",
776 "display_name": "Python 3 (ipykernel)",
777 "language": "python",
785 "file_extension": ".py",
786 "mimetype": "text/x-python",
788 "nbconvert_exporter": "python",
789 "pygments_lexer": "ipython3",