runtime/BlackBoxRuntime.cpp

   1 #include "BlackBoxRuntime.hpp"
   2 #include <mysql++/mysql++.h>
   3
   4 BlackBoxCache* gBlackBoxCache = NULL;
   5
   6 char*
   7 BBArun_wchar_to_UTF8(const wchar_t *str)
   8 {
   9   uint32_t len = 0;
  10   const wchar_t* p = str;
  11   wchar_t c;
  12   for (; (c = *p); p++)
  13   {
  14     if (c <= 0x7F)
  15       len++;
  16     else
  17 #ifndef WCHAR_T_IS_32BIT
  18     if (c <= 0x7FF)
  19 #endif
  20       len += 2;
  21 #ifndef WCHAR_T_IS_32BIT
  22     else if (c <= 0xFFFF)
  23       len += 3;
  24     else
  25       len += 4;
  26 #endif
  27   }
  28   char* newData = (char*)malloc(len + 1);
  29   char* np = newData;
  30   p = str;
  31
  32   while ((c = *p++))
  33   {
  34     if (c <= 0x7F)
  35       *np++ = (char)c;
  36     else if (c <= 0x7FF)
  37     {
  38       *np++ = (char)(0xC0 | ((c >> 6) & 0x1F));
  39       *np++ = (char)(0x80 | (c & 0x3F));
  40     }
  41 #ifndef WCHAR_T_CONSTANT_WIDTH
  42     else if ((c & 0xFC00) == 0xD800)
  43     {
  44       uint16_t u = ((c >> 6) & 0xF) + 1;
  45       *np++ = 0xF0 | ((u >> 2) & 0x7);
  46       *np++ = 0x80 | ((u << 4) & 0x30) | ((c >> 2) & 0xF);
  47       wchar_t c2 = *p++;
  48       *np++ = 0x80 | ((c << 4) & 0x30) | ((c2 >> 6) & 0xF);
  49       *np++ = 0x80 | (c2 & 0x3F);
  50     }
  51 #endif
  52     else
  53 #if defined(WCHAR_T_CONSTANT_WIDTH) && !defined(WCHAR_T_IS_32BIT)
  54          if (c <= 0xFFFF)
  55 #endif
  56     {
  57       *np++ = (char)(0xD0 | ((c >> 12) & 0xF));
  58       *np++ = (char)(0x80 | ((c >> 6) & 0x3F));
  59       *np++ = (char)(0x80 | (c & 0x3F));
  60     }
  61 #if defined(WCHAR_T_CONSTANT_WIDTH) && !defined(WCHAR_T_IS_32BIT)
  62     else
  63     {
  64       *np++ = (char)(0xF0 | ((c >> 18) & 0x7));
  65       *np++ = (char)(0x80 | ((c >> 12) & 0x3F));
  66       *np++ = (char)(0x80 | ((c >> 6) & 0x3F));
  67       *np++ = (char)(0x80 | (c & 0x3F));
  68     }
  69 #endif
  70   }
  71
  72   *np++ = 0;
  73   return newData;
  74 }
  75
  76 class SQLModelPersister
  77   : public BlackBoxPersistBase
  78 {
  79 public:
  80   SQLModelPersister()
  81     : mConn(mysqlpp::use_exceptions), mReady(false)
  82   {
  83     const char* db = getenv("BLABOC_DB");
  84     if (db == NULL)
  85     {
  86       printf("Missing BLABOC_DB. Not creating persister.\n");
  87       return;
  88     }
  89     const char* host = getenv("BLABOC_HOST");
  90     if (host == NULL)
  91     {
  92       printf("Missing BLABOC_HOST. Not creating persister.\n");
  93       return;
  94     }
  95     const char* user = getenv("BLABOC_USER");
  96     if (user == NULL)
  97     {
  98       printf("Missing BLABOC_USER. Not creating persister.\n");
  99       return;
 100     }
 101     const char* pass = getenv("BLABOC_PASSWORD");
 102     if (pass == NULL)
 103     {
 104       printf("Missing BLABOC_PASSWORD. Not creating persister.\n");
 105       return;
 106     }
 107
 108     try
 109     {
 110       mConn.connect(db, host, user, pass);
 111       mReady = true;
 112     }
 113     catch (std::exception& e)
 114     {
 115       printf("Error connecting to database: %s. Not creating persister.\n",
 116              e.what());
 117     }
 118   }
 119
 120   ~SQLModelPersister()
 121   {
 122   }
 123
 124   void
 125   saveState(const char* aURL, const std::string& aData)
 126   {
 127     if (!mReady)
 128       return;
 129
 130     // Figure out if we need to update or insert...
 131     mysqlpp::Query query(mConn.query());
 132
 133     query << "SELECT COUNT(*) FROM black_box_state WHERE url=\""
 134           << mysqlpp::escape << aURL << "\"";
 135
 136     try
 137     {
 138       mysqlpp::Result res = query.store();
 139       unsigned int count = res.at(0)[(unsigned int)0];
 140       query.reset();
 141       if (count == 0)
 142       {
 143         query << "INSERT INTO black_box_state (url,data) VALUES (\""
 144               << mysqlpp::escape << aURL << "\", \""
 145               << mysqlpp::escape << aData << "\")";
 146         query.execute();
 147       }
 148       else
 149       {
 150         query << "UPDATE black_box_state SET data=\""
 151               << mysqlpp::escape << aData << "\" WHERE url=\""
 152               << mysqlpp::escape << aURL << "\"";
 153         query.execute();
 154       }
 155     }
 156     catch (std::exception& e)
 157     {
 158       printf("Exception saving state: %s\n", e.what());
 159     }
 160   }
 161
 162   void
 163   restoreState(const char* aURL, std::string& aData)
 164   {
 165     if (!mReady)
 166     {
 167       aData.assign("");
 168       return;
 169     }
 170
 171     mysqlpp::Query query(mConn.query());
 172     query << "SELECT data FROM black_box_state WHERE url=\""
 173           << mysqlpp::escape << aURL << "\"";
 174
 175     try
 176     {
 177       mysqlpp::Result res = query.store();
 178       if (res.rows() == 0)
 179       {
 180         aData.assign("");
 181         return;
 182       }
 183       mysqlpp::ColData col = res.at(0)["data"];
 184       aData.assign(col.data(), col.size());
 185     }
 186     catch (std::exception& e)
 187     {
 188       printf("Exception restoring state: %s\n", e.what());
 189       aData.assign("");
 190     }
 191   }
 192
 193 private:
 194   mysqlpp::Connection mConn;
 195   bool mReady;
 196 };
 197
 198 MixedModelBase::MixedModelBase()
 199   : mPersister(new SQLModelPersister()),
 200     VARIABLES(NULL), INPUTS(NULL), OUTPUTS(NULL),
 201     CONSTANTS(NULL), TMPINPUTS(NULL), TMPOUTPUTS(NULL),
 202     blackBoxes(NULL), blackBoxNames(NULL),
 203     BACKVARIABLES(NULL), BACKOUTPUTS(NULL)
 204 {
 205 }
 206
 207 MixedModelBase::~MixedModelBase()
 208 {
 209   size_t i;
 210
 211   if (BACKVARIABLES != NULL)
 212     delete [] BACKVARIABLES;
 213
 214   if (BACKOUTPUTS != NULL)
 215     delete [] BACKOUTPUTS;
 216
 217   if (blackBoxes != NULL)
 218   {
 219     std::string state;
 220
 221     if (gBlackBoxCache == NULL)
 222     {
 223       for (i = 0; i < nBlackBoxes; i++)
 224         if (blackBoxes[i] != NULL)
 225         {
 226           blackBoxes[i]->Serialise(state);
 227           if (state.size() != 0)
 228             mPersister->saveState(blackBoxNames[i].c_str(), state);
 229           delete blackBoxes[i];
 230         }
 231     }
 232     delete [] blackBoxes;
 233   }
 234
 235   delete mPersister;
 236
 237   if (blackBoxNames != NULL)
 238     delete [] blackBoxNames;
 239 }
 240
 241 bool
 242 MixedModelBase::runConverged()
 243 {
 244   if (mConvergeSteps++ > 10)
 245   {
 246     printf("Warning: It took more than 10 steps for the black-boxes to "
 247            "converge within tolerance. Quitting early.\n");
 248     return true;
 249   }
 250
 251   uint32_t i, l = countOtherVariables();
 252   double RSS = 0, t;
 253
 254 #if 0
 255   printf("Computing RSS with...\n");
 256   for (i = 0; i < countOtherVariables(); i++)
 257     printf(" * OTHERVARIABLES[%u] = %g\n", i, VARIABLES[i]);
 258   for (i = 0; i < countOutputVariables(); i++)
 259     printf(" * OUTPUTS[%u] = %g\n", i, OUTPUTS[i]);
 260   printf("Listing complete.\n");
 261 #endif
 262
 263   for (i = 0; i < l; i++)
 264   {
 265     t = VARIABLES[i] - BACKVARIABLES[i];
 266     RSS += t * t;
 267   }
 268   l = countOutputVariables();
 269   for (i = 0; i < l; i++)
 270   {
 271     t = OUTPUTS[i] - BACKOUTPUTS[i];
 272     RSS += t * t;
 273   }
 274
 275 #if 0
 276   printf("Step %u: RSS = %g\n", mConvergeSteps, RSS);
 277 #endif
 278
 279   if (RSS < THRESHOLD)
 280     return true;
 281
 282   return false;
 283 }
 284
 285 void
 286 MixedModelBase::createBlackBox
 287 (
 288  uint32_t aIdx,
 289  const wchar_t* URI,
 290  uint32_t inputs,
 291  uint32_t outputs
 292 )
 293 {
 294   if (gBlackBoxCache)
 295   {
 296     BlackBoxModelBase* cachedModel = gBlackBoxCache->FindModelInCache(URI);
 297     if (cachedModel != NULL)
 298     {
 299       blackBoxes[aIdx] = cachedModel;
 300
 301       char* utf8uri = BBArun_wchar_to_UTF8(URI);
 302       blackBoxNames[aIdx] = utf8uri;
 303       free(utf8uri);
 304
 305       return;
 306     }
 307   }
 308
 309   const wchar_t* offs = wcschr(URI, '#');
 310   std::wstring name, id;
 311   if (offs == NULL)
 312     name = URI;
 313   else
 314     name = std::wstring(URI, offs - URI);
 315
 316   if (offs != NULL)
 317     id = offs + 1;
 318
 319   std::map<std::wstring, BlackBoxModelFactory*>::iterator i
 320     = blackBoxFactories().find(name);
 321
 322   if (i == blackBoxFactories().end())
 323   {
 324     printf("Warning: Cannot create a black box with URI base %S because "
 325            "no factory could be found.\n",
 326            name.c_str());
 327     throw std::exception();
 328   }
 329
 330   blackBoxes[aIdx] = (*i).second->createBlackBoxModel(id, inputs, outputs);
 331   char* utf8uri = BBArun_wchar_to_UTF8(URI);
 332   blackBoxNames[aIdx] = utf8uri;
 333   free(utf8uri);
 334
 335   std::string data;
 336   mPersister->restoreState(blackBoxNames[aIdx].c_str(), data);
 337   if (data != "")
 338     blackBoxes[aIdx]->Deserialise(data);
 339
 340   if (gBlackBoxCache != NULL)
 341     gBlackBoxCache->CacheModel(URI, blackBoxes[aIdx]);
 342 }
 343
 344 std::map<std::wstring,BlackBoxModelFactory*>* MixedModelBase::sBlackBoxFactories = NULL;
 345
 346 class StandardBlackBoxCache
 347   : public BlackBoxCache
 348 {
 349 public:
 350   StandardBlackBoxCache()
 351   {
 352   }
 353
 354   ~StandardBlackBoxCache()
 355   {
 356     DiscardCache();
 357   }
 358
 359   BlackBoxModelBase*
 360   FindModelInCache(const std::wstring& aURI)
 361   {
 362     std::map<std::wstring, BlackBoxModelBase*>::iterator i = mModels.find(aURI);
 363
 364     if (i == mModels.end())
 365       return NULL;
 366
 367     return (*i).second;
 368   }
 369
 370   void
 371   CacheModel(const std::wstring& aURI, BlackBoxModelBase* aModel)
 372   {
 373     mModels.insert(std::pair<std::wstring, BlackBoxModelBase*>(aURI, aModel));
 374   }
 375
 376   void
 377   SaveAllCachedModels()
 378   {
 379     std::map<std::wstring, BlackBoxModelBase*>::iterator i;
 380     std::string state;
 381
 382     SQLModelPersister smp;
 383
 384     for (i = mModels.begin(); i != mModels.end(); i++)
 385     {
 386       (*i).second->Serialise(state);
 387       if (state.size() != 0)
 388       {
 389         char* utf8uri = BBArun_wchar_to_UTF8((*i).first.c_str());
 390         smp.saveState(utf8uri, state);
 391         free(utf8uri);
 392       }
 393     }
 394   }
 395
 396   void
 397   DiscardCache()
 398   {
 399     std::map<std::wstring, BlackBoxModelBase*>::iterator i;
 400     for (i = mModels.begin(); i != mModels.end(); i++)
 401       delete (*i).second;
 402     mModels.clear();
 403   }
 404 private:
 405   std::map<std::wstring, BlackBoxModelBase*> mModels;
 406 };
 407
 408 void setupStandardBlackBoxCache()
 409 {
 410   gBlackBoxCache = new StandardBlackBoxCache();
 411 }