óraállás-diktáló/nkm-aramszunet.js

   1 #!/usr/bin/env phantomjs2 --ssl-protocol=any --ignore-ssl-errors=true --web-security=false
   2
   3 /*
   4         EXIT CODE
   5         0       report successful
   6         1       a web interaction step failed on user side
   7         2       wrong invocation
   8         3       page load failed
   9         4       a web interaction step failed on navigator side
  10         5       report failed
  11 */
  12
  13 var llDebug = 1, llVerbose = 2, llError = 3, llMsg = 4;
  14 var vlDebug = false;
  15 var vlVerbose = false;
  16 var vlError = true;
  17 var vlMsg = true;
  18 var system = require('system');
  19 var stderr = system.stderr;
  20 var stdout = system.stdout;
  21 var Glob = {};
  22
  23 for(var i=1; i<system.args.length; i++)
  24 {
  25         if(system.args[i] == '--debug') vlDebug = true;
  26         else if(system.args[i] == '--verbose') vlVerbose = true;
  27         else if(system.args[i].match(/^-/))
  28         {
  29                 stderr.write("Unknown option: "+system.args[i]+"\n");
  30                 phantom.exit(2);
  31         }
  32         else
  33         {
  34                 if(!Glob.telepules)
  35                 {
  36                         Glob.telepules = system.args[i];
  37                 }
  38                 else if(!Glob.date_from)
  39                 {
  40                         Glob.date_from = system.args[i];
  41                 }
  42                 else if(!Glob.date_till)
  43                 {
  44                         Glob.date_till = system.args[i];
  45                 }
  46         }
  47 }
  48
  49 if(!Glob.telepules || !Glob.date_from || !Glob.date_till)
  50 {
  51         stderr.write("Usage: nkm-aramszunet.js [--debug] [--verbose] <település> <dátum-tól> <dátum-ig>\n");
  52         phantom.exit(2);
  53 }
  54
  55
  56 var url_form = "https://www.mvmnext.hu/aram/pages/online/aramszunet.jsf";
  57 var loadInProgress = false;
  58 var stepindex = 0;
  59 var page = new WebPage();
  60 page.settings.userAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.90 Safari/537.36 Vivaldi/1.91.867.38";
  61
  62 function log(level, msg)
  63 {
  64         if(level == llDebug && vlDebug ||
  65            level == llVerbose && vlVerbose ||
  66            level == llError && vlError ||
  67            level == llMsg && vlMsg)
  68         {
  69                 stderr.write(msg + "\n");
  70         }
  71 }
  72
  73 page.onConsoleMessage = function(msg)
  74 {
  75         if(msg.match(/^\[DEBUG\]/))
  76                 log(llDebug, msg);
  77         else if(msg.match(/^\[ERROR\]/))
  78                 log(llError, msg);
  79         else
  80                 log(llMsg, ">>> " + msg);
  81 };
  82
  83 page.onLoadStarted = function()
  84 {
  85         loadInProgress = true;
  86         log(llVerbose, "load started [" + page.url + "]");
  87 };
  88
  89 page.onLoadFinished = function()
  90 {
  91         loadInProgress = false;
  92         log(llVerbose, "load finished <" + page.evaluate(function(){return document.title;}) + "> [" + page.url + "]");
  93 };
  94
  95 page.onError = function(msg, stack)
  96 {
  97         log(llError, "ERROR: " + msg + "\nSTACK: " + JSON.stringify(stack));
  98 };
  99
 100 page.onResourceRequested = function(requestData, networkRequest)
 101 {
 102         /* allow only these domains, no trackers, no ads */
 103         var allow = false;
 104         var g = requestData.url.match(/^.+?:\/\/(.+?)(\/[^\?]*)/);
 105         var host = g[1];
 106         var path = g[2];
 107         var hostpath = host+path;
 108         // google maps seem to needed for the page to work properly, but I worked it around
 109         //if(hostpath == 'maps.google.com/maps/api/js') allow = true;
 110         if(host == 'www.mvmnext.hu') allow = true;
 111         if(!allow)
 112         {
 113                 log(llDebug, "deny: " + requestData.method + " " + requestData.url);
 114                 networkRequest.abort();
 115         }
 116
 117         if(requestData.url.match(/^https?:\/\/(maps\.google\.com)\//))
 118         {
 119                 // mock out some base parts of gmaps lib
 120                 page.evaluate(function(){
 121                         window.google = {};
 122                         google.maps = {};
 123                         google.maps.MapTypeId = {};
 124                         google.maps.MapTypeId.ROADMAP = {};
 125                         google.maps.LatLng = function(){ return {}; };
 126                         google.maps.InfoWindow = function(){ return {}; };
 127                         google.maps.Map = function(){ return {
 128                                 'getBounds': function(){ return {}; },
 129                         }; };
 130                         google.maps.event = { 'addListener': function(){ return {}; }, };
 131                         google.maps.Marker = function(){ return {
 132                                 'setMap': function(){ return {}; },
 133                         }; };
 134                         console.log("gmaps mocked.");
 135                 });
 136         }
 137 };
 138
 139
 140 Glob.inject = 'function click(elem){'+
 141 '       var ev = document.createEvent("MouseEvent");'+
 142 '       ev.initMouseEvent("click", true /* bubble */, true /* cancelable */, window, null, 0, 0, 0, 0, /* coordinates */ false, false, false, false, /* modifier keys */ 0 /*left*/, null);'+
 143 '       elem.dispatchEvent(ev);'+
 144 '};'+
 145 'function grab_output(){'+
 146 '       return document.querySelector(\'[id="aramszunetForm:aramszunetTable_data"]\').innerHTML;'+
 147 '};'+
 148 'function grab_aramszunetTable(){'+
 149 '       return document.querySelector(\'[id="aramszunetForm:aramszunetTable"]\').innerHTML;'+
 150 '};';
 151
 152
 153
 154 page.open(url_form, function(status)
 155 {
 156         if(status !== 'success')
 157         {
 158                 log(llError, "ERROR " + status);
 159                 phantom.exit(3);
 160         }
 161         else
 162         {
 163
 164                 var steps = [
 165                         function(param)
 166                         {
 167                                 /* Waiting page to load */
 168                                 var input_telepules = document.querySelector('select[id="aramszunetForm:telepules_input"]');
 169                                 if(!input_telepules)
 170                                 {
 171                                         //console.log(document.querySelectorAll('html')[0].outerHTML);
 172                                         console.log("Waiting for 'Település' field...");
 173                                         return 0;
 174                                 }
 175                                 return 1;
 176                         },
 177                         function(param)
 178                         {
 179                                 // save the empty 'no results' table for comparasion later on
 180                                 eval(param.inject);
 181                                 var noResults = grab_aramszunetTable();
 182                                 var rv = {'result': {'aszt': noResults}, 'jump': 1};
 183                                 return JSON.stringify(rv);
 184                         },
 185                         function(param)
 186                         {
 187                                 console.log("submitting form...");
 188                                 var telepules_element = document.querySelector('li[data-label="' + param.telepules + '"]');
 189                                 if(!telepules_element)
 190                                 {
 191                                         var msg = "crawler: A település '" + param.telepules + "' nincs a tervezett áramszünetek listájában."
 192                                         console.log(msg);
 193                                         var output = "<tr><td>"+msg+"</td></tr>";
 194                                         return JSON.stringify({'result': {'output': output}, 'jump': 'end'});
 195                                 }
 196                                 eval(param.inject);
 197                                 /* Filling in form fields and submit */
 198                                 click(telepules_element);
 199                                 document.querySelector('input[id="aramszunetForm:startDate_input"]').value = param.date_from;
 200                                 document.querySelector('input[id="aramszunetForm:endDate_input"]').value = param.date_till;
 201
 202                                 // 3rd party components are celled via this method,
 203                                 // ignore errors in 3rd parties (errors occur due to not loaded 3rd party scripts)
 204                                 PrimeFaces.ajax.ResponseProcessor.doEval = function(b){
 205                                         var a = b.textContent || b.innerText || b.text;
 206                                         try { $.globalEval(a) }
 207                                         catch(e) { console.log('Exception', e); }
 208                                 };
 209                                 // the onClick event handler on the submit button looks like this:
 210                                 //PrimeFaces.ab({s:"aramszunetForm:j_idt29",p:"aramszunetForm",u:"aramszunetForm:aramszunetTable aramszunetForm:filters"});
 211
 212                                 var submit_btn = document.querySelector('#aramszunetForm button');
 213                                 click(submit_btn);
 214                                 return 1;
 215                         },
 216                         function(param)
 217                         {
 218                                 // there were many trial-and-error here to find out how to set rows-per-page which PrimeFaces understands
 219                                 console.log("setting paginator rows per page to 1000...");
 220                                 var rppdd = document.querySelector('select[name="aramszunetForm:aramszunetTable_rppDD"]');
 221                                 var opt = document.createElement('option');
 222                                 opt.value = 1000;
 223                                 opt.innerText = "1000";
 224                                 rppdd.appendChild(opt);
 225                                 //rppdd.selectedIndex = rppdd.options.length - 1;
 226                                 //opt.setAttribute('selected', 'selected');
 227                                 rppdd.value = opt.value;
 228                                 //rppdd.setAttribute('value', opt.value);
 229                                 //$(rppdd).val(opt.value);
 230                                 PrimeFaces.widgets.aramszunetTable.paginator.setRowsPerPage(opt.value);
 231                                 //console.log("rppdd=", $(rppdd).val(), rppdd.value, rppdd.getAttribute('value'));
 232                                 //opt.selected = true;
 233                                 //$(rppdd).val(opt.value);$(rppdd).change();
 234
 235                                 //var ev = document.createEvent("HTMLEvents");
 236                                 //ev.initEvent('change', true /* bubble */, true /* cancelable */);
 237                                 //rppdd.dispatchEvent(ev);
 238                                 return 1;
 239                         },
 240                         function(param)
 241                         {
 242                                 eval(param.inject);
 243                                 console.log("waiting 'aramszunetTable' to change...");
 244                                 var aszt = grab_aramszunetTable();
 245                                 var rv = {'result': {'aszt': aszt}, 'jump': 0};
 246                                 if(param.aszt != rv.result.aszt) {
 247                                         rv.jump = 1;
 248                                 }
 249                                 return JSON.stringify(rv);
 250                         },
 251                         function(param)
 252                         {
 253                                 eval(param.inject);
 254                                 var rv = {'result': {'output': grab_output()}, 'jump': 1};
 255                                 return JSON.stringify(rv);
 256                         }
 257                 ];
 258
 259                 Glob.total_steps = steps.length;
 260
 261                 timeout = 500;
 262                 worker = function()
 263                 {
 264                         if(!loadInProgress)
 265                         {
 266                                 /* Execute next step in navigator's context */
 267                                 log(llDebug, "step " + stepindex);
 268                                 var ret = page.evaluate(steps[stepindex], Glob);
 269
 270                                 if(ret === '')
 271                                 {
 272                                         phantom.exit(1);
 273                                 }
 274                                 else if(ret === undefined || ret === null)
 275                                 {
 276                                         phantom.exit(4);
 277                                 }
 278
 279                                 if(typeof ret != "number")
 280                                 {
 281                                         ret = JSON.parse(ret);
 282                                         for(var key in ret.result)
 283                                         {
 284                                                 Glob[key] = ret.result[key];
 285                                         }
 286                                         ret = ret.jump;
 287                                 }
 288
 289                                 log(llDebug, "step " + stepindex + " jump " + ret);
 290                                 if(ret == 'end') stepindex = steps.length;
 291                                 else stepindex += ret;
 292
 293                                 if(stepindex >= steps.length)
 294                                 {
 295                                         stdout.write(Glob.output);
 296                                         log(llDebug, "completed");
 297                                         phantom.exit(0);
 298                                 }
 299                         }
 300
 301                         setTimeout(worker, timeout);
 302                 }
 303                 setTimeout(worker, timeout);
 304         }
 305 });