Merge branch 'lua_versions' into main/rendor-staging

[ryzomcore.git] / ryzom / tools / sheet_random_generator / srg_utilities.h

// Ryzom - MMORPG Framework <http://dev.ryzom.com/projects/ryzom/>
// Copyright (C) 2010  Winch Gate Property Limited
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.



#ifndef NL_SRG_UTILITIES_H
#define NL_SRG_UTILITIES_H


// Misc
#include <nel/misc/types_nl.h>
#include <nel/misc/sstring.h>
#include "nel/misc/path.h"
#include "nel/misc/file.h"
#include "nel/misc/smart_ptr.h"
#include "nel/misc/command.h"
#include "nel/misc/common.h"
#include "nel/misc/path.h"
#include <nel/misc/diff_tool.h>
#include <nel/misc/random.h>
// Georges
#include "nel/georges/u_form.h"
#include "nel/georges/u_form_elm.h"
#include "nel/georges/u_form_dfn.h"
#include "nel/georges/u_form_loader.h"
#include "nel/georges/u_type.h"
// Georges, bypassing interface
#include "georges/stdgeorges.h"
#include "georges/form.h"
// C
#include <time.h>
#include <conio.h>
// stl
#include <set>
#include <map>
#include <hash_map>

using namespace NLMISC;
using namespace NLGEORGES;
using namespace std;


typedef sint TFaberInterestLevel;
const TFaberInterestLevel NAInterestLevel = -1;
const uint32 NbNomenclaturedFaberLevel = 5; // excluding N/A
const char *sNomenclaturedInterestLevels [NbNomenclaturedFaberLevel+1] = { "N/A", "Worst", "Bad", "Average", "Good", "Best" }; // from -1 to 5
CRandom RandomGenerator;

typedef CVectorSString vs;

typedef map <uint32, vector<uint32> > CRulesFilter;
typedef map <CSString, vector<uint32>, CUnsensitiveSStringLessPred > CRulesStrFilter;
typedef map <CSString, vs, CUnsensitiveSStringLessPred > CRulesStr2Filter;
typedef map< CSString, CSString, CUnsensitiveSStringLessPred > mss;
typedef vector<uint32> vu;
typedef hash_map< string, string, hash<string> > CTitles; // code -> utf8 title

// Write sheet files, or display to screen only
bool WriteSheetsToDisk = true;

// Overriden by command-line argument after -n
string ExtractNamesCsv;


const uint32 NbFaberElements = 26;

const uint32 RM_INDEX_FAMILY_CODE = 1;
const uint32 NB_FAMILY_CODE_CHARS = 4;

const uint32 RM_INDEX_CREATURE_CODE = 5;
const uint32 NB_CREATURE_CODE_CHARS = 5;

const uint32 RM_INDEX_ECOSYSTEM_CODE = 8;
const uint32 NB_ECOSYSTEM_CODE_CHARS = 1;

const uint32 RM_INDEX_LEVELZONE_CODE = 9;

const uint32 SIZE_RAW_MATERIAL_SHEET_FILENAME = 18;

/*const uint32 RM_INDEX_INTEREST_LEVEL = 6;
const uint32 NB_INTEREST_LEVEL_CHARS = 2;

const uint32 RM_INDEX_FABERELEMS_CODE = 8;
const uint32 NB_FABERELEMS_CODE_CHARS = 6;*/

const uint32 CR_INDEX_ECOSYSTEM_CODE = 3; // in creature code
const uint32 CR_INDEX_LEVELZONE_CODE = 4;
const uint32 CR_INDEX_LOCAL_LEVEL_CODE = 5;

//const uint32 NB_UNIQUE_LEVELS_PER_ZONE = 4; // not counting the bosses, the fifth equals the first one of next 
const uint32 NB_UNIQUE_LEVELZONES_PER_CONTINENT = 5;
const uint32 MAX_NB_LOCAL_LEVELS = 8;
//const uint32 NB_ZONE_LEVELS = 5; // TEMP for forest ecosystem
//const uint32 NbFaberInterestLevels = (NB_ZONE_LEVELS * NB_UNIQUE_LEVELS_PER_ZONE) + 1; // excluding N/A

string                          inputSheetPath;
bool                            inputSheetPathLoaded = false;
map<string, string>     inputSheetPathContent; // short filename without ext, full filename with path
string                          TranslationPath;
string                          SystemMPPath;


// These vectors have the same indices : by family
vs families;
vs familyCodes, ecosystemCodes, propertyCodes, creatureCodes;

sint MaxFamilyNum = -1;

enum TColor { Red, Beige, Green, Turquoise, Blue, Violet, White, Black, NbColors, InvalidColor=NbColors };

// By ecosystem, color, property, rm group, creature, season
vs ecosystems, colors, properties, groups, creatures, seasons;
mss adjectives;

// DtName must be the 1st one
enum TDataCol { DtName, DtTitle, DtRMFamily, DtGroup, DtEcosystem, DtLevelZone, DtStatQuality, DtProp, DtCreature, DtCreaTitle, DtCraftSlotName, DtCraftCivSpec, DtColor, DtAverageEnergy, DtMaxLevel, DtJewelProtectionType, DtCustomizedProperties, DtNbCols };
const char *DataColStr [DtNbCols] = { "Code", "Name", "Family", "Group", "Ecosystem", "LevelZone", "Stat Quality", "Properties", "Creature sheets", "Creatures", "Item parts", "Craft civ spec", "Color", "Average energy", "Max level", "Jewel protection type", "Customized" };

const uint32 NbPropertyDepths = 5;
const char *PropertyDepths [NbPropertyDepths] = { "Unknown", "Slightly", "Moderately", "Quite", "Extremely" };

typedef uint32 TGroup; // index in groups, and value in groups .typ

//enum TLocation { Under, Over, Flora, Creatures, NB_LOCATIONS, NB_DEPOSITS=Flora+1 };
enum TLocation { InDeposits, InCreatures, NB_LOCATIONS };
vu DepositFamilyIndices;

enum TCiv { Fyros, Matis, Tryker, Zorai, AllCiv, NbCiv };
const char *CivNames [NbCiv] = { "Fyros", "Matis", "Tryker", "Zorai", "All" };
const char *CivEcosystemCodes [NbCiv] = { "D", "F", "L", "J", "X" };

enum TEcosystem { CommonEcosystem, Desert, Forest, Lacustre, Jungle, PrimeRoots, NbEcosystems, Goo=NbEcosystems, Invasion, Raid, Event, N, S, T, U, V, W, X, Y, Z, NbEcosystemsPlusExtensions };

TCiv EcosystemToCiv[NbEcosystems] = { AllCiv, Fyros, Matis, Tryker, Zorai, AllCiv };

enum TStatQuality { Basic, Fine, Choice, Excellent, Supreme, NbStatQualities, InvalidStatQuality=NbStatQualities };
char *StatQualityStr [NbStatQualities+1] = { "Basic", "Fine", "Choice", "Excellent", "Supreme", "N/A" };

TStatQuality CreatureLocalLevelToStatQuality [MAX_NB_LOCAL_LEVELS] =
{
        Basic,                          // 1 (index 0)
        Fine,                           // 2 
        Basic,                          // 3
        Fine,                           // 4
        Excellent,                      // 5 (named creatures)
        InvalidStatQuality,     // 6 (mission creatures, their RMs have no craft stats)
        Supreme,                        // 7 (bosses)
        Choice                          // 8 (mini-bosses)
};


enum TIndexOfRemarkableStatIndex { RBestA, RBest=RBestA, RWorstA1, RWorst1=RWorstA1, RWorstA2, RWorst2=RWorstA2, RBestB, RWorstB1, RWorstB2, NB_REMARKABLE_STAT_INDICES };

struct TSkeletonInfo
{
        TSkeletonInfo() : IsUsed(false) {}

        //vs                            CreaturesOfSke;
        CSString                        Name;
        CSString                        AbbrevName;
        //CSString                      SkGroup;
        bool                            IsUsed; // true if has RM
};

typedef map<CSString, TSkeletonInfo, CUnsensitiveSStringLessPred> CSkeletonMap;

// By skeleton group, by skeleton, bu creature model
CRulesStr2Filter                SkgroupToModels;
CSkeletonMap                    CreatureModels;
mss                                             CreatureToModel;
//vs                                            skeletonGroupColumns;
set<CSString, CUnsensitiveSStringLessPred>                      CreatureMainModelsWithoutRM;

// By creature model
CRulesStrFilter RMFamilyIndicesByCreatureModel;
vu GooCreatureFamilyIndices;

typedef map<char, vu> CCreatureTypeToFamilyIndices; // the key is the 2nd char of the creature code (h for herbivore, k for kitin...)
CCreatureTypeToFamilyIndices InvasionRaidCreatureFamilyIndices;

struct TBool
{
        TBool() : Done(false) {}
        bool Done;
};

typedef map< uint, TBool > CDoneMap; // indexed by levelzone
map< string, TBool > IsRMSheetGenerated; // indexed by sheetname

string rawMaterialPath, creaturePath, creatureAssignmentPath, depositPath;
string dirbase;


const string oldRmSheetType = "item";
const string rmSheetType = "sitem";
const string crSheetType = "creature";
const string dpSheetType = "deposit";


//const uint32 NbPropertySlots = 10; // obsolete
uint32 UndefinedProperty = ~0;



/*
 *
 */
struct CIconInfo
{
        CSString        IconBackground, Icon, IconOver, IconOver2;
};

map< CSString, CIconInfo, CUnsensitiveSStringLessPred > Icons;


/*
 *
 */
class CMainStat
{
public:

        ///
        CMainStat() : SumNbFaberElemsFilled(0), MaxNbFaberElemsFilled(0), MinNbFaberElemsFilled(~0),
                NbRMByFaberElem( NbFaberElements, 0 ) {}

        /// Call it when families etc. are ready
        void    init()
        {
                NbRMByFaberElemByFamilyAndCiv.resize( families.size() );
                for ( uint32 i=0; i!=families.size(); ++i )
                {
                        NbRMByFaberElemByFamilyAndCiv[i].resize( NbCiv );
                        for ( uint32 c=0; c!=NbCiv; ++c )
                                NbRMByFaberElemByFamilyAndCiv[i][c].resize( NbFaberElements );
                }

                for ( uint32 iEcosystem=0; iEcosystem!=ecosystems.size(); ++iEcosystem )
                {
                        for ( uint32 iLoc=0; iLoc!=NB_LOCATIONS; ++iLoc )
                        {
                                NbRMHavingProperty[iLoc][iEcosystem].resize( properties.size() );
                                NbRMByFaberElemByEcosystem[iEcosystem].resize( NbFaberElements );
                        }
                }
        }

        ///
        bool    updateCraftStatistics( uint32 rFaberElem, uint32 iEcosystem, uint32 iFam, TCiv civ );

        ///
        void    updateMainStats( uint32 nbFaberElemsFilled )
        {
                SumNbFaberElemsFilled += nbFaberElemsFilled;
                if ( nbFaberElemsFilled < MinNbFaberElemsFilled )
                        MinNbFaberElemsFilled = nbFaberElemsFilled;
                if ( nbFaberElemsFilled> MaxNbFaberElemsFilled )
                        MaxNbFaberElemsFilled = nbFaberElemsFilled;
        }

        // By property and by ecosystem and location
        vu NbRMHavingProperty [NB_LOCATIONS][NbEcosystems];

        uint32 SumNbFaberElemsFilled;
        uint32 MaxNbFaberElemsFilled;
        uint32 MinNbFaberElemsFilled;

        vector<uint32> NbRMByFaberElem;;
        vector<uint32> NbRMByFaberElemByEcosystem [NbEcosystems];
        vector< vector < vector< uint32 > > > NbRMByFaberElemByFamilyAndCiv;

};


/*
 * From georges2csv
 */
void    loadSheetPath()
{
        if (inputSheetPathLoaded)
                return;

        NLMISC::createDebug();
        NLMISC::WarningLog->addNegativeFilter( "CPath::insertFileInMap" );

        CPath::addSearchPath(inputSheetPath, true, false); // for Georges to work properly

        vector<string>  files;
        CPath::getPathContent (inputSheetPath, true, false, true, files);

        uint32  i;
        for (i=0; i<files.size(); ++i)
        {
                string  filename = files[i];
                string  filebase = CFile::getFilenameWithoutExtension(filename);
                inputSheetPathContent[filebase] = filename;
        }

        inputSheetPathLoaded = true;
}


/*
 * Get a random value in range [0..nbPossibleValues[
 * Precondition: nbPossibleValues < 65536
 */
inline uint32 getRandomValue( uint32 nbPossibleValues )
{
        /*double        r = (double) rand();
        r/= (double) (RAND_MAX+1); // exclude the ceiling value
        return (uint32)(r * nbPossibleValues);*/
        return RandomGenerator.rand( (uint16)(nbPossibleValues-1) );
}


/*
 *
 */
struct CDfnFieldInfo
{
        CDfnFieldInfo() {}
        CDfnFieldInfo( const vector<string>& values, const vector<string>& labels ) : TypePredefinedValues(values), TypePredefinedLabels(labels) {}

        vector<string>  TypePredefinedValues;
        vector<string>  TypePredefinedLabels;
};


/*
 *
 */
struct TIconMapping
{
        const char *FamilyName;
        const char *IconFilename;
};


/*
 *
 */
sint getNomenclaturedInterestLevel( TFaberInterestLevel level, TFaberInterestLevel nbInterestLevels )
{
        return (level == NAInterestLevel) ? 0 : (level * NbNomenclaturedFaberLevel / nbInterestLevels) + 1;
}


/*
 *
 */
//struct CFaberCode
//{
//      char Ch[NB_FABERELEMS_CODE_CHARS];
//};


/*
 *
 */
//struct CFaberCombination
//{
//      CFaberCombination( TFaberInterestLevel firstLevel, const string& code ) : FirstLevel(firstLevel)
//      {
//              memcpy( Code.Ch, &code[0], NB_FABERELEMS_CODE_CHARS );
//      }
//
//      TFaberInterestLevel     FirstLevel;
//      CFaberCode                      Code;
//};


/*
 *
 */
//class CSheetNameRepository
//{
//public:
//
//      ///
//      void    resize( uint32 nbEcosystems, uint32 nbFamilies )
//      {
//              _Container.resize( nbEcosystems );
//              for ( uint32 i=0; i!=nbEcosystems; ++i )
//              {
//                      _Container[i].resize( nbFamilies );
//              }
//      }
//
//      ///
//      void    insert( uint32 iEcosystem, uint32 iFamily, uint32 iCreature, TFaberInterestLevel level, const string& faberCombinationCode )
//      {
//              // nlassert( faberCombinationCode.size() == NB_FABERELEMS_CODE_CHARS );
//              _Container[iEcosystem][iFamily][iCreature].push_back( CFaberCombination( level, faberCombinationCode ) );
//      }
//
//      ///
//      void    getFaberCombinationCodes( uint32 iEcosystem, uint32 iFamily, uint32 iCreature, vector<CFaberCombination> **codes )
//      {
//              map < uint32, vector< CFaberCombination > >::iterator im;
//              /*nldebug( "%u %u -> %u cr (searching for %u)", iEcosystem, iFamily, _Container[iEcosystem][iFamily].size(), iCreature );
//              for ( im=_Container[iEcosystem][iFamily].begin(); im!=_Container[iEcosystem][iFamily].end(); ++im )
//                      nldebug( "cr %u -- %u combinations", (*im).first, (*im).second.size() );*/
//              im = _Container[iEcosystem][iFamily].find( iCreature );
//              if ( im == _Container[iEcosystem][iFamily].end() )
//                      *codes = NULL;
//              else
//                      *codes = &((im)->second);
//      }
//
//private:
//
//      /// Indexs: iEcosystem, iFamily, rCreatureSpecialization, rFaberCombination
//      vector< vector < map < uint32, vector< CFaberCombination > > > >        _Container;
//};


//void  CSheetNameRepository::getFaberCombinationCodes( uint32 iEcosystem, uint32 iFamily, uint32 iCreature, vector<CFaberCombination> **codes )


//CSheetNameRepository RawMaterialRepository;

/**
 * Characteristics.
 * When adding a new characteristic, ADD IT INTO "v3_source_tables.xls!Item Parts v3"/"rm_item_parts.csv" and mark compatible item parts
 */
enum    TFaberCharacteristic {
        Durability, Weight, SapLoad, DMG, Speed, Range,
        DodgeModifier, ParryModifier, AdversaryDodgeModifier, AdversaryParryModifier,
        ProtectionFactor, MaxSlashProtect, MaxBluntProtect, MaxPierceProtect,
        ECTF, EPF,
        OACTF, OAPF,
        HCTP, HPF,
        DACTF, DAPF, 
        JewelProtection,
        CraftCivSpec,
        NbCharacs };

const char *sCharacs [NbCharacs] = {
        "Durability", "Weight", "SapLoad", "DMG", "Speed", "Range",
        "DodgeModifier", "ParryModifier", "AdversaryDodgeModifier", "AdversaryParryModifier",
        "ProtectionFactor", "MaxSlashingProtection", "MaxBluntProtection", "MaxPiercingProtection",
        "ElementalCastingTimeFactor", "ElementalPowerFactor",
        "OffensiveAfflictionCastingTimeFactor", "OffensiveAfflictionPowerFactor",
        "HealCastingTimeFactor", "HealPowerFactor",
        "DefensiveAfflictionCastingTimeFactor", "DefensiveAfflictionPowerFactor",
        "JewelProtectionType",
        "CraftCivSpec" };

//const bool PositiveCharacs [NbCharacs] = {    true,   false,  true,   false,  true,   true };
//const float MinCharacValues [NbCharacs] = {   100,    0.1f,   10,             0.2f,   100,    0 };
//const float MaxCharacValues [NbCharacs] = {   500,    1.5f,   400,    2.0f,   500,    60 };
//const float PeakCharacValues [NbCharacs] = {  2000,   2.5f,   800,    5.0f,   2000,   80 };
bool PositiveCharacs [NbCharacs];
float MinCharacValues [NbCharacs];
float MaxCharacValues [NbCharacs];
float PeakCharacValues [NbCharacs];
vector<bool> CharacSlotFilter [NbCharacs]; // it's a positive filter



/*
 *
 */
struct TFamInfo
{
        vs                              Properties;                             // ex: propForA, propForB, propForC
        CSString                CompatibleCraftParts;   // ex: ABC
        vu                              CraftPartsByProp;               // ex: 0, 1, 2 (indices in ompatibleCraftParts)
        vector<TCiv>    Civs;                                   // ex: Fyros, All, All
        vu                              Freqs;                                  // ex: 1, 2, 2
        TGroup                  Group;
        bool                    IsActive;                                       // False if not in rm_fam_prop.csv
        bool                    IsInDeposits;
        bool                    IsInCreatures;
        CSString                        SpecialCreatureTag;
        bool                    GenerateOnly;
        bool                    IsForMission;
        sint8                   RemarkableStatIndex [NB_REMARKABLE_STAT_INDICES];
        sint8                   ColorIndex;
        sint8                   JewelProtIndex;

        static uint             UseGenerateOnly; // if 0, generate all; otherwise, generate only families that have GenerateOnly set to true

        ///
        TFamInfo() : Group(~0), IsInDeposits(false), IsActive(false), IsInCreatures(false), SpecialCreatureTag(), GenerateOnly(false), IsForMission(false), ColorIndex(-1), JewelProtIndex(-1) {}

        ///
        uint                    getCompatibleCraftPart( uint iCompatibleCP ) const
        {
                return (uint)(CompatibleCraftParts[iCompatibleCP] - 'A');
        }

        /// whichProp: index in Properties
        uint                    getBeginCraftPartForProp( uint whichProp ) const
        {
                return CraftPartsByProp[whichProp];
        }

        /// whichProp: index in Properties
        uint                    getEndCraftPartForProp( uint whichProp ) const
        {
                if ( whichProp == Properties.size()-1 )
                        return CompatibleCraftParts.size();
                else
                        return CraftPartsByProp[whichProp+1];
        }

        /// whichProp: index in Properties
        CSString                getCraftPartForProp( uint whichProp ) const
        {
                uint start = getBeginCraftPartForProp( whichProp );
                return CompatibleCraftParts.substr( start, getEndCraftPartForProp( whichProp ) - start );
        }

        /** With the returned index, you can get elt in Property, CraftPartsByProp, Civs and Freqs.
         * Returns ~0 if not found.
         */
        uint                    getPropIndexByCraftPart( char itemPart ) const
        {
                for ( uint i=0; i!=CraftPartsByProp.size(); ++i )
                {
                        char itemPartCode [2];
                        itemPartCode[0] = itemPart;
                        itemPartCode[1] = '\0';
                        if ( getCraftPartForProp( i ).find( itemPartCode ) != string::npos )
                                return i;
                }
                return ~0;
        }

        ///
        bool                    existsInEcosystem( TEcosystem iEcosystem, TStatQuality statQuality ) const;

        ///
        static bool             mustGenerateFamily( uint iFamily );
};


const uint ITEM_PART_JEWEL_GEM = 17; // R


/*
 *
 */
struct CFaberCharacteristics
{
        /// Default constructor (for reading)
        CFaberCharacteristics()
                : FaberElement(~0), ActualEnergy(0.0f), ActualOriginality(0.0f)
        {
                for ( uint32 i=0; i!=NbCharacs; ++i )
                        Values[i] = 0.0f;
        }

        ///
        void    serial( NLMISC::IStream& s )
        {
                s.serial( FaberElement );
                s.serial( ActualEnergy );
                s.serial( ActualOriginality );
                for ( uint32 c=0; c!=NbCharacs; ++c )
                        s.serial( Values[c] );
        }

        ///
        void    initFaberElement( uint32 rFaberElement ) { FaberElement = rFaberElement; }
        
        /// Returns false if the RM must NOT be generated.
        bool    randomizeValues( TFaberInterestLevel interestLevel, TFaberInterestLevel nbInterestLevels, uint iVariant, float widthRatio, float peakOccurRatio, float baseBoost, TEcosystem iEcosystem, uint iFreq, uint iFamily );

        /// Returns -1 if the RM must NOT be generated, otherwise return the stat average
        sint32  computeValues( TStatQuality statQuality, const TFamInfo& famInfo, sint remarkableIndicesSetBaseIndex,
                                                   TEcosystem iEcosystem, uint iFreq, uint iFamily );

        ///
        void    calcQualitativeValues();

        /// Index of faber element in "MpParam"
        uint32  FaberElement;

        /// Values
        float   Values [NbCharacs];

        /// Average of the actual interest of the random values between 0 and 1
        float   ActualEnergy;

        ///
        float   ActualOriginality;

protected:

        // Returns false if the RM must NOT be generated.
        //bool  randomizeJewelProtection( TStatQuality statQuality, TEcosystem iEcosystem, uint iFreq, uint iFamily );

        void    computeJewelProtection( TStatQuality statQuality, TEcosystem iEcosystem, uint iFamily );
};


//
void    CFaberCharacteristics::calcQualitativeValues()
{
        float actualInterests [NbCharacs];

        // Calculate the average of ratio between [0,1]
        float sumActualInterest = 0.0f;
        uint nbCharacsUsed = 0;
        for ( uint r=0; r!=NbCharacs; ++r )
        {
                if ( MaxCharacValues[r] == 0 )
                        continue;
                if ( ! CharacSlotFilter[r][FaberElement] )
                        continue;

                float interestRatio = (Values[r] / PeakCharacValues[r]); // new: Peak is taken as the max => the Energy is in [0..100]
                if ( ! PositiveCharacs[r] )
                        interestRatio = 1.0f - interestRatio; // thus, can be negative

                actualInterests[r] = interestRatio;
                sumActualInterest += interestRatio;
                ++nbCharacsUsed;
        }

        if ( nbCharacsUsed == 0 )
                return;

        ActualEnergy = (sumActualInterest / (float)nbCharacsUsed);
        if ( ActualEnergy > 1.0f )
                ActualEnergy = 1.0f;

        // Calculate the standard deviation (SQRT(SUM((Ai-Aavg)^2)/N))
        float varianceSum = 0.0f;
        for ( uint r=0; r!=NbCharacs; ++r )
        {
                if ( MaxCharacValues[r] == 0 )
                        continue;
                if ( ! CharacSlotFilter[r][FaberElement] )
                        continue;

                varianceSum += sqr( actualInterests[r] - ActualEnergy );
        }

        // Don't normalize standard deviation, otherwise low energy materials will be considered more
        // original (by average) than high energy materials. They wouldn't be comparable.

        //if ( ActualEnergy != 0.0f )
                ActualOriginality = (float)sqrt( (double)(varianceSum / (float)nbCharacsUsed) ); // / ActualEnergy;
        //else
        //      nlinfo( "Null energy for craft slot %u", FaberElement );
}


/*
 * Fill childrenToGet if rootNameForGetChildren is not null
 */
void fillFromDFN( UFormLoader *formLoader, map<string, CDfnFieldInfo>& dfnFields, UFormDfn *formDfn, const string& rootName, const string& dfnFilename,
                                  const char *rootNameForGetChildren=NULL, vector<string>& childrenToGet=vector<string>() )
{
        uint32 i;
        for ( i=0; i!=formDfn->getNumEntry(); ++i )
        {
                string entryName, rootBase;
                formDfn->getEntryName( i, entryName );
                rootBase = rootName.empty() ? "" : (rootName+".");

                UFormDfn::TEntryType entryType;
                bool array;
                formDfn->getEntryType( i, entryType, array );
                switch ( entryType )
                {
                        case UFormDfn::EntryVirtualDfn:
                        {
                                CSmartPtr<UFormDfn> subFormDfn = formLoader->loadFormDfn( (entryName + ".dfn").c_str() );
                                if ( ! subFormDfn )
                                        nlwarning( "Can't load virtual DFN %s", entryName.c_str() );
                                else
                                        fillFromDFN( formLoader, dfnFields, subFormDfn, rootBase + entryName, entryName + ".dfn", rootNameForGetChildren, childrenToGet ); // recurse
                                break;
                        }
                        case UFormDfn::EntryDfn: // .dfn
                        {
                                UFormDfn *subFormDfn;
                                if ( formDfn->getEntryDfn( i, &subFormDfn) )
                                {
                                        string filename;
                                        formDfn->getEntryFilename( i, filename );
                                        fillFromDFN( formLoader, dfnFields, subFormDfn, rootBase + entryName, filename, rootNameForGetChildren, childrenToGet ); // recurse
                                }
                                if ( rootNameForGetChildren && (rootName == rootNameForGetChildren) )
                                {
                                        childrenToGet.push_back( rootBase + entryName );
                                }
                                break;
                        }
                        case UFormDfn::EntryType: // .typ
                        {
                                vector<string> values, labels;
                                UType *subType;
                                if ( formDfn->getEntryType( i, &subType ) )
                                {
                                        uint32 listSize = subType->getNumDefinition();
                                        if ( listSize > 0 )
                                        {
                                                string label, value;
                                                for ( uint32 j=0; j!=listSize; ++j )
                                                {
                                                        subType->getDefinition( j, label, value );
                                                        if ( (subType->getIncrement() == "1") && (subType->getType() == UType::UnsignedInt || subType->getType() == UType::SignedInt) )
                                                        {
                                                                // Fill blank entry for skipped identifier values (to allow indexing by identifier value)
                                                                sint num = atoi( value.c_str() );
                                                                while ( num - (sint)values.size() > 0 )
                                                                {
                                                                        values.push_back( "" );
                                                                        labels.push_back( "" );
                                                                }
                                                        }
                                                        values.push_back( value );
                                                        labels.push_back( label );
                                                }
                                        }
                                }
                                dfnFields.insert( make_pair( rootBase + entryName, CDfnFieldInfo(values, labels) ) );
                                //nlinfo( "DFN entry: %s (in %s)", (rootBase + entryName).c_str(), dfnFilename.c_str() );
                                break;
                        }
                }
        }
}


/*
 *
 */
CForm *loadTemplateForm( UFormLoader *formLoader, const string& sheetType )
{
        CForm *form = (CForm*)formLoader->loadForm( (string("_empty.")+sheetType).c_str() );
        if ( ! form )
                nlerror( "Can't load sheet _empty.%s", sheetType.c_str() );
        return form;
}


/*
 *
 */
void eraseCarriageReturns( string& s )
{
        const char CR = '\n';
        string::size_type p = s.find( CR );
        while ( (p=s.find( CR )) != string::npos )
                s.erase( p, 1 );
}


/*
 *
 */
string getNomenclatureCode( const string& longName, set<string>& usedCodes, uint32 nbLetters )
{
        if ( nbLetters > longName.size() )
                nlerror( "Wrong nbLetters for %s", longName.c_str() );

        // Start with beginning of name
        string code = strlwr(longName.substr( 0, nbLetters ));
        uint32 i = nbLetters-1;
        while ( usedCodes.find( code ) != usedCodes.end() )
        {
                ++i;
                if ( i < longName.size() )
                {
                        // Substitute last code char by a char from the name (except ' ')
                        if ( longName[i] != ' ' )
                                code[nbLetters-1] = tolower(longName[i]);
                        else
                                continue;
                }
                else
                {
                        // If no char from the name is suitable, increment the last char of the code until suitable
                        char c=1;
                        while ( usedCodes.find( code ) != usedCodes.end() )
                        {
                                code[nbLetters-1] = tolower(longName[nbLetters-1]) + c;
                                ++c;
                                if ( code[1] > 'z' )
                                        nlerror( "Impossible to make code for %s", longName.c_str() );
                        }
                }
        }
        strlwr( code );
        usedCodes.insert( code );
        return code;
}


/*
 * Displays mapping if title not null.
 */
void buildNomenclatureCodes( const char *title, const vector<string>& longNames, vector<string>& codes, uint32 nbLetters )
{
        set<string> usedCodeSet;
        uint32 i;
        for ( i=0; i!=longNames.size(); ++i )
        {
                codes[i] = getNomenclatureCode( longNames[i], usedCodeSet, nbLetters );
                if ( title )
                        nlinfo( "%s %s -> %s", title, longNames[i].c_str(), codes[i].c_str() );
                //DebugLog->displayRawNL( "%s", longNames[i].c_str() );
        }
}


/*
 * Set the size of a family code to NB_FAMILY_CODE_CHARS
 */
void normalizeFamilyCode( std::string& s )
{
        if ( s.size() > NB_FAMILY_CODE_CHARS )
        {
                nlerror( "Family codes limited to %u chars (%s)", NB_FAMILY_CODE_CHARS, s.c_str() );
        }
        else
        {
                uint p = s.size();
                while ( p < NB_FAMILY_CODE_CHARS )
                {
                        s = "0" + s;
                        ++p;
                }
        }
}


/*
 *
 */
void loadNomenclatureCodes( const char *title, const vector<string>& longNames, vector<string>& codes, const char *filename )
{
        if ( longNames.empty() )
        {
                nlwarning( "No nomenclature codes to load. %s", title ? title : "" );
                return;
        }
        codes.resize( longNames.size() );

        char lineBuffer[2048];
        FILE *rulesFile;
        const char *SEPARATOR = ";";
        vector<string> args;
        vector<string>::iterator iarg;
        vector<string>::const_iterator ivs;

        if ( (rulesFile = NLMISC::nlfopen( filename, "r" )) == NULL )
        {
                nlwarning( "Can't find file %s", filename );
        }
        else
        {
                while ( ! feof(rulesFile) )
                {
                        // Get from file
                        fgets( lineBuffer, 2048, rulesFile );
                        explode( lineBuffer, SEPARATOR, args );

                        // Get rid of carriage returns!
                        for ( iarg=args.begin(); iarg!=args.end(); ++iarg )
                        {
                                eraseCarriageReturns( *iarg );
                        }

                        // Read
                        const uint32 MIN_COLS = 6;
                        const uint32 NAME_COL = 4;
                        const uint32 C_COL = 2;
                        const uint32 R_COL = 3;
                        const uint32 NB_CODE_CHARS = 2;
                        if ( (args.size()>=MIN_COLS) && (! args[0].empty()) && (args[0].find( "name" )==string::npos) ) // skip blank lines, and lines with blank header or "name" in the first column
                        {
                                if ( args[NAME_COL].empty() )
                                        continue;

                                ivs = find( longNames.begin(), longNames.end(), args[NAME_COL] );
                                if ( ivs == longNames.end() )
                                        nlwarning( "Name %s is not in the names array", args[NAME_COL].c_str() );
                                else
                                {
                                        string code = args[C_COL] + args[R_COL];
                                        if ( code.size() < NB_CODE_CHARS )
                                        {
                                                nlwarning( "Invalid partial code for %s: %s", (*ivs).c_str(), code.c_str() );
                                                continue;
                                        }
                                        else if ( code.size() > NB_CODE_CHARS )
                                        {
                                                nlinfo( "Compacting code '%s' for %s", code.c_str(), (*ivs).c_str() );
                                                string::size_type p;
                                                while ( (p = code.find( ' ' )) != string::npos )
                                                {
                                                        code.erase( p, 1 );
                                                }
                                        }

                                        if ( codes[ivs-longNames.begin()].empty() )
                                        {
                                                if ( title )
                                                        nlinfo( "%s %s -> %s", title, (*ivs).c_str(), code.c_str() );
                                                codes[ivs-longNames.begin()] = code;
                                        }
                                        else
                                        {
                                                if ( code != codes[ivs-longNames.begin()] )
                                                        nlwarning( "Invalid nomenclature: (%s and %s for %s: ", codes[ivs-longNames.begin()].c_str(), code.c_str(), (*ivs).c_str() );
                                        }
                                }
                        }
                }

                for ( ivs=codes.begin(); ivs!=codes.end(); ++ivs )
                {
                        if ( (*ivs).empty() )
                                nlwarning( "No code found for %s", (*(longNames.begin() + (ivs - codes.begin()))).c_str() );
                }
        }
}


/*
 *
 */
inline sint getLastUsedPropertySlot( uint32 *iProperties, sint lastPropertySlot, uint32 undefinedProperty )
{
        for ( sint r=lastPropertySlot; r>=0; --r )
        {
                if ( iProperties[r] != undefinedProperty )
                        return r;
        }
        return -1;
}


/*
 *
 */
inline bool passNegativeFilter( const vector<uint32>& incompatibilityList, uint32 iValue )
{
        vector<uint32>::const_iterator ip = find( incompatibilityList.begin(), incompatibilityList.end(), iValue );     
        return (ip == incompatibilityList.end());
}


/*
 *
 */
inline bool passPositiveFilter( const vector<uint32>& compatibilityList, uint32 iValue )
{
        vector<uint32>::const_iterator ip = find( compatibilityList.begin(), compatibilityList.end(), iValue ); 
        return (ip != compatibilityList.end());
}


/*
 * Reject a prop if it is in the incompatibility list of a family
 */
bool passPropFamilyFilter( const vector<uint32>& iFamilyRelatedProperties, uint32 iProp )
{
        return passNegativeFilter( iFamilyRelatedProperties, iProp );
}


/*
 * Reject a creature if NOT in the creature list of a family
 */
/*bool passCreatureFilter( const vector<uint32>& iFamilyRelatedCreatures, uint32 iCreature )
{
        //nldebug( "%u related creatures, %s", iFamilyRelatedCreatures.size(), passPositiveFilter( iFamilyRelatedCreatures, iCreature ) ? "TRUE": "FALSE" );
        return passPositiveFilter( iFamilyRelatedCreatures, iCreature );
}*/


/*
 *
 */
class CStrIComparator : public binary_function<string, string, bool>
{
public:
        bool    operator() ( const string& s1, const string& s2 ) const
        {
                return (nlstricmp( s1, s2 ) == 0);
        }
};


/*
 *
 */
void displayList( const vector<string>& v, CLog *log=DebugLog )
{
        vector<string>::const_iterator ist;
        for ( ist=v.begin(); ist!=v.end(); ++ist )
                log->displayRaw( "%s ", (*ist).c_str() );
        log->displayRawNL( "" );
}


/*
 *
 */
uint32 getIndexFromString( const string& s, const vector<string>& v, bool displayWarning=true )
{
        if ( v.empty() )
        {
                if ( displayWarning )
                        nlwarning( "Can't find '%s' in empty array", s.c_str() );
                return ~0;
        }
        else
        {
                vector<string>::const_iterator ist = find_if( v.begin(), v.end(), bind2nd(CStrIComparator(), s) );
                if ( ist == v.end() )
                {
                        if ( displayWarning )
                        {
                                nlwarning( "Can't find '%s' in:", s.c_str() );
                                displayList( v, WarningLog );
                        }
                        return ~0;
                }
                else
                        return ist - v.begin();
        }
}


/*
 *
 */
uint32 getIndexFromString( const string& s, const char **array, uint arraySize, bool displayWarning=true )
{
        if ( arraySize == 0 )
        {
                if ( displayWarning )
                        nlwarning( "Can't find '%s' in empty array", s.c_str() );
                return ~0;
        }
        else
        {
                for ( uint i=0; i!=arraySize; ++i )
                {
                        if ( strlwr(string(array[i])) == strlwr(s) )
                                return i;
                }

                if ( displayWarning )
                {
                        nlwarning( "Can't find '%s' in:", s.c_str() );
                        //displayList( v, WarningLog );
                }
                return ~0;
        }
}


/*
 * Returns the index of the erased element, ~0 if not found
 */
uint32 removeEntryFromList( vector<string>& v, const string& entry )
{
        vector<string>::iterator ivs;
        ivs = find( v.begin(), v.end(), entry );
        uint32 index;
        if ( ivs != v.end() )
        {
                index = ivs - v.begin();
                v.erase( ivs );
                return index;
        }
        else
                return ~0;
}


/*
 *
 */
bool removeEntryFromListByIndex( vector<string>& v, uint32 index )
{
        if ( index < v.size() )
        {
                v.erase( v.begin() + index );
                return true;
        }
        else
                return false;
}


typedef void (*TMapDeliveryCallback) ( mss& );
typedef void (*TVectorDeliveryCallback) ( vs& );


/*
 *
 */
void    loadCSVFile( const char *filename, TMapDeliveryCallback deliveryCallback, bool firstColWithoutName=false )
{
        char lineBuffer[2048];
        FILE *file;
        const char *SEPARATOR = ";";
        vector<string> args;
        vector<string>::iterator iarg;

        if ( (file = NLMISC::nlfopen( filename, "r" )) == NULL )
        {
                nlwarning( "Can't find file %s", filename );
        }
        else
        {
                // Read first line as header with column names
                lineBuffer[0] = '\0';
                fgets( lineBuffer, 2048, file );
                explode( lineBuffer, SEPARATOR, args );

                // Store column names (and get rid of carriage returns!)
                vector < string > columnNames;
                mss valuesByName;
                for ( iarg=args.begin(); iarg!=args.end(); ++iarg )
                {
                        if ( firstColWithoutName && (iarg == args.begin()) )
                        {
                                *iarg = "<>"; // override column name for the 1st column
                        }
                        eraseCarriageReturns( *iarg );
                        columnNames.push_back( *iarg );
                        valuesByName.insert( make_pair( *iarg, string("") ) );
                }

                while ( ! feof(file) )
                {
                        // Get from file
                        lineBuffer[0] = '\0';
                        fgets( lineBuffer, 2048, file );
                        explode( lineBuffer, SEPARATOR, args );

                        // Set values (and get rid of carriage returns!)
                        for ( iarg=args.begin(); iarg!=args.end(); ++iarg )
                        {
                                eraseCarriageReturns( *iarg );
                                valuesByName[columnNames[iarg-args.begin()]] = *iarg;
                        }

                        // Deliver the wanted fields
                        deliveryCallback( valuesByName );
                }
        }
}


/*
 *
 */
void    loadCSVFile( const char *filename, TVectorDeliveryCallback deliveryCallback )
{
        char lineBuffer[2048];
        FILE *file;
        const char *SEPARATOR = ";";
        vs args;
        vs::iterator iarg;

        if ( (file = NLMISC::nlfopen( filename, "r" )) == NULL )
        {
                nlwarning( "Can't find file %s", filename );
        }
        else
        {
                while ( ! feof(file) )
                {
                        // Get from file
                        lineBuffer[0] = '\0';
                        fgets( lineBuffer, 2048, file );
                        explode( lineBuffer, SEPARATOR, args );

                        // Get rid of carriage returns!
                        for ( iarg=args.begin(); iarg!=args.end(); ++iarg )
                        {
                                eraseCarriageReturns( *iarg );
                        }

                        // Deliver the wanted fields
                        deliveryCallback( args );
                }
        }
}


/*
 *
 */
void loadValueFile( const char *filename, const vector<string>& keyStrings,
                                        vector<sint>& contents, sint defaultValue )
{
        nlassert( keyStrings.size() == contents.size() );
        char lineBuffer[2048];
        FILE *rulesFile;
        const char *SEPARATOR = ";";
        vector<string> args;
        vector<string>::iterator iarg;

        if ( (rulesFile = NLMISC::nlfopen( filename, "r" )) == NULL )
        {
                nlwarning( "Can't find file %s", filename );
        }
        else
        {
                while ( ! feof(rulesFile) )
                {
                        // Get from file
                        lineBuffer[0] = '\0';
                        fgets( lineBuffer, 2048, rulesFile );
                        explode( lineBuffer, SEPARATOR, args );

                        // Get rid of carriage returns!
                        for ( iarg=args.begin(); iarg!=args.end(); ++iarg )
                        {
                                eraseCarriageReturns( *iarg );
                        }

                        // Read
                        if ( (! args.empty()) && (! args[0].empty()) ) // skip blank lines, and lines with blank header
                        {
                                sint value = defaultValue;
                                for ( uint32 a=0; a!=args.size()-1; ++a )
                                {
                                        if ( ! args[a+1].empty() ) // skip blank entries
                                                value = atoi( args[a+1].c_str() );
                                }
                                uint32 index = getIndexFromString( args[0], keyStrings );
                                if ( index != ~0 )
                                {
                                        contents[index] = value;
                                } 
                        }
                }
                fclose( rulesFile );
        }
}


/*
 *
 */
void loadRulesFile( const char *filename, const vector<string>& keyStrings,
                                    const vector<string>& contentStrings, CRulesFilter& filter,
                                        const string& matchExtKeyAtFirstColumn=string() )
{
        char lineBuffer[2048];
        FILE *rulesFile;
        const char *SEPARATOR = ";";
        uint32 firstColumn = matchExtKeyAtFirstColumn.empty() ? 0 : 1;
        vector<string> args;
        vector<string>::iterator iarg;

        if ( (rulesFile = NLMISC::nlfopen( filename, "r" )) == NULL )
        {
                nlwarning( "Can't find file %s", filename );
        }
        else
        {
                while ( ! feof(rulesFile) )
                {
                        // Get from file
                        lineBuffer[0] = '\0';
                        fgets( lineBuffer, 2048, rulesFile );
                        explode( lineBuffer, SEPARATOR, args );

                        // Get rid of carriage returns!
                        for ( iarg=args.begin(); iarg!=args.end(); ++iarg )
                        {
                                eraseCarriageReturns( *iarg );
                        }

                        // Match with ext key string if set
                        if ( (! matchExtKeyAtFirstColumn.empty()) && (args[0]!=matchExtKeyAtFirstColumn) )
                                continue;

                        // Read
                        if ( (! args.empty()) && (! args[firstColumn].empty()) ) // skip blank lines, and lines with blank header
                        {
                                vector<uint32> contents;
                                for ( uint32 a=firstColumn; a!=args.size()-1; ++a )
                                {
                                        if ( ! args[a+1].empty() ) // skip blank entries
                                                contents.push_back( getIndexFromString( args[a+1], contentStrings ) );
                                }
                                filter.insert( make_pair( getIndexFromString( args[firstColumn], keyStrings ), contents ) );
                        }
                }
                fclose( rulesFile );
        }
}


/*
 * 1st column: extKeyStrings (corresponding to the filters 'vector'); 2nd: keyStrings
 */
void loadRulesFileMulti( const char *filename, const vector<string>& extKeyStrings, const vector<string>& keyStrings, const vector<string>& contentStrings, vector<CRulesFilter>& filters )
{
        filters.resize( extKeyStrings.size() );
        for ( uint32 i=0; i!=filters.size(); ++i )
        {
                loadRulesFile( filename, keyStrings, contentStrings, filters[i], extKeyStrings[i] );
                /*CRulesFilter::const_iterator irf;
                nldebug( "%s", extKeyStrings[i].c_str() );
                for ( irf=filters[i].begin(); irf!=filters[i].end(); ++irf )
                {
                        nldebug( "%s", keyStrings[(*irf).first].c_str() );
                        vector<uint32>::const_iterator ivi;
                        for ( ivi=(*irf).second.begin(); ivi!=(*irf).second.end(); ++ivi )
                        {
                                nldebug( "%u", *ivi );
                        }
                }*/
        }
}


/*
 * Clear the form to reuse it (and all contents below node)
 */
void clearSheet( CForm *form, UFormElm* node )
{
        ((CFormElm*)node)->clean();
        form->clean();
}


/*
 * Saves to disk if bool WriteSheetsToDisk is true 
 */
void flushSheetToDisk( const string& fullFilename, UForm *form )
{
        if ( WriteSheetsToDisk )
        {
                COFile output( fullFilename );
                form->write( output, false );
        }
}


/*
 *
 */
string::size_type findCapital( const string& s, string::size_type startPos )
{
        string::size_type p;
        for ( p=startPos; p!=s.size(); ++p )
        {
                if ( (s[p] >= 'A') && (s[p] <= 'Z') )
                        return p;
        }
        return string::npos;
}


/*
 * Transform "MyString " into "My string"
 */
void detachValue( string& s )
{
        if ( s.size() < 2 )
                return;

        string::size_type p;
        while ( (p = findCapital( s, 1 )) != string::npos )
        {
                s.insert( p, " " );
                s[p+1] = tolower( s[p+1] );
        }

        // Rip off any blank at the end
        if ( s[s.size()-1] == ' ' )
        {
                s.resize( s.size()-1 );
        }
}


/*
 *
 */
void getTransposedMap( CRulesFilter& dest, const CRulesFilter& src )
{
        CRulesFilter::const_iterator im;
        for ( im=src.begin(); im!=src.end(); ++im )
        {
                vector<uint32>::const_iterator iv;
                for ( iv=(*im).second.begin(); iv!=(*im).second.end(); ++iv )
                {
                        dest[*iv].push_back( (*im).first );
                }
        }
}


/*
 *
 */
string makeFaberElementCode( uint32 iFaberElement, TFaberInterestLevel level, TFaberInterestLevel nbInterestLevels )
{
        return toString( "%c%d", 'a' + iFaberElement, getNomenclaturedInterestLevel( level, nbInterestLevels ) );
}


/*
 *
 */
inline bool hasMatchingFaberLevel( TFaberInterestLevel storedLevel, TFaberInterestLevel submittedLevel )
{
        return storedLevel <= submittedLevel;
}


/*
 *
 */
//void keepOnlyHighestLevel( vector<CFaberCombination*>& codes )
//{
//      nlassert( ! codes.empty() );
//      sint maxLevel = -1;
//      uint32 i;
//      for ( i=0; i!=codes.size(); ++i )
//      {
//              if ( codes[i]->FirstLevel > maxLevel )
//              {
//                      maxLevel = codes[i]->FirstLevel;
//              }
//      }
//      vector<CFaberCombination*> remainingCodes;
//      for ( i=0; i!=codes.size(); ++i )
//      {
//              if ( codes[i]->FirstLevel == maxLevel )
//                      remainingCodes.push_back( codes[i] );
//      }
//
//      //nldebug( "%u codes, highest level = %u with %u occurences", codes.size(), maxLevel, remainingCodes.size() );
//      //nlassert( remainingCodes.size() <= codes.size() );
//      codes = remainingCodes;
//      nlassert( ! codes.empty() );
//}


/*
 *
 */
bool    allIncludedIn( const vu& subset, const vu& bigset )
{
        vu::const_iterator iv;
        for ( iv=subset.begin(); iv!=subset.end(); ++iv )
        {
                if ( find( bigset.begin(), bigset.end(), *iv ) == bigset.end() )
                        return false;
        }
        return true;
}


/*
 *
 */
void loadConfigFlag( CConfigFile& configFile, const char *varTitle, bool &flag )
{
        CConfigFile::CVar *var = configFile.getVarPtr( varTitle );
        if ( var )
                flag = (var->asInt() == 1);
}


/*
 *
 */
string::size_type getCapitalFromPos( const string& s, string::size_type startPos )
{
        //nldebug( "%s %u", s.c_str(), startPos );
        string::size_type p;
        for ( p=startPos; p<s.size(); ++p )
        {
                if ( (s[p] >= 'A') && (s[p] <= 'Z') )
                        return p;
        }
        return string::npos;
}


/*
 * Also used to make system_mp filenames.
 * Converts "My Identifier" or "MyIdentifier" to "my_identifier" ("My identifier" to "Myidentifier")
 */
string conventionalDirectory( const string& dirname )
{
        if ( dirname.empty() )
                return "";

        string result = dirname;

        // Remove blanks
        string::size_type p = 0;
        while ( (p = result.find( ' ' )) != string::npos )
        {
                result.erase( p, 1 );
        }

        // Convert capitals to underscores
        result[0] = tolower( result[0] );
        p = 1;
        while ( (p = getCapitalFromPos( result, p )) != string::npos )
        {
                result.insert( p, "_" );
                ++p;
                result[p] = tolower( result[p] );
        }
        return result;
}


mss     UniqueRMNamesAndSheetCodeHead;


/*
 *
 */
void    readRMNames( mss& values )
{
        string& name = values["basics.name"];
        if ( ! name.empty() )
        {
                string radix = values["FILE"].substr( 0, 5 );
                UniqueRMNamesAndSheetCodeHead.insert( make_pair( name, radix ) );
        }
}


/*
 *
 */
void    loadTitles( const string& sourceWords, const string& sourceBase, const string& languageCode, CTitles& dest )
{
        STRING_MANAGER::TWorksheet worksheet;
        STRING_MANAGER::loadExcelSheet( TranslationPath + sourceBase + "/" + sourceWords + "_words_" + languageCode + ".txt", worksheet );
        uint cp, cn, nbTitles = 0;
        if ( worksheet.findCol( ucstring(sourceWords + " ID"), cp ) && worksheet.findCol( ucstring("name"), cn ) )
        {
                for ( std::vector<STRING_MANAGER::TWorksheet::TRow>::iterator ip = worksheet.begin(); ip!=worksheet.end(); ++ip )
                {
                        if ( ip == worksheet.begin() ) // skip first row
                                continue;
                        STRING_MANAGER::TWorksheet::TRow& row = *ip;
                        dest.insert( make_pair( row[cp].toString(), row[cn].toUtf8() ) );
                        ++nbTitles;
                }
        }
        else
                nlwarning( "%s ID or name not found", sourceWords.c_str() );

        nlinfo( "Loaded %u %s titles", nbTitles, sourceWords.c_str() );
}


/*
 *
 */
void    extractRawMaterialNames()
{
        loadCSVFile( ExtractNamesCsv.c_str(), readRMNames );
        FILE *output = NLMISC::nlfopen( CFile::getFilenameWithoutExtension( ExtractNamesCsv ) + "_output.csv", "wt" );
        fprintf( output, "Code;Name\n" );
        for ( mss::const_iterator iun=UniqueRMNamesAndSheetCodeHead.begin(); iun!=UniqueRMNamesAndSheetCodeHead.end(); ++iun )
        {
                const string& codeRadix = (*iun).second;
                const string& name = (*iun).first;
                fprintf( output, "%s;%s\n", codeRadix.c_str(), name.c_str() );
        }
}


/*
 *
 */
void    cleanExteriorWhitespace( vs& line )
{
        for ( vs::iterator it=line.begin(); it!=line.end(); ++it )
        {
                CSString& s = (*it);
                string::size_type p;
                for ( p=0; p!=s.size(); ++p )
                {
                        if ( s[p] != ' ' )
                                break;
                }
                if ( (p != 0) && (p != s.size()) )
                        s = s.substr( p );

                for ( p=0; p!=s.size(); ++p )
                {
                        if ( s[s.size()-1-p] != ' ' )
                                break;
                }
                if ( (p != 0) && (p != s.size()) )
                        s = s.rightCrop( p );
        }
}


uint TFamInfo::UseGenerateOnly = 0;

// Only used for deposits; for creature, works with the creature sheets found
/*bool TFamInfo::existsInEcosystem( TEcosystem iEcosystem, TStatQuality statQuality ) const
{
        switch ( iEcosystem )
        {
        case CommonEcosystem: // The Common rm exists if the rm family has a freq=2 (or 0 but only in Supreme)
                return ( (find( Freqs.begin(), Freqs.end(), 2 ) != Freqs.end())
                          || ((find( Freqs.begin(), Freqs.end(), 0 ) != Freqs.end()) && (statQuality == Supreme)) );
                          // was: find_if ... bind2nd( equals<uint>(), 1 )
                break;
        case PrimeRoots: // Only freq 1 families exist if the PrimeRoots
                return find( Freqs.begin(), Freqs.end(), 1 ) != Freqs.end();
                break;
        default: // A rm family exists in the ecosystem matching a civ if the corresponding freq is 1 or 3
                {
                uint iCiv = getIndexFromString( ecosystemCodes[iEcosystem], CivEcosystemCodes, NbCiv, false );
                vector<TCiv>::const_iterator it = find( Civs.begin(), Civs.end(), (TCiv)iCiv );
                if ( it != Civs.end() )
                        return (Freqs[it-Civs.begin()] == 1) || (Freqs[it-Civs.begin()] == 3);
                else
                        return false;
                }
        }
}*/

// Only used for deposits;
bool TFamInfo::existsInEcosystem( TEcosystem iEcosystem, TStatQuality statQuality ) const
{
        if ( find( Freqs.begin(), Freqs.end(), 0 ) != Freqs.end() )
        {
                // Freq 0 => only Common/Supreme
                return (statQuality == Supreme) && (iEcosystem == CommonEcosystem);
        }
        else if ( statQuality <= Fine )
        {
                // Basic, Fine => Common
                return (iEcosystem == CommonEcosystem);
        }
        else
        {
                // Choice to Supreme => One per ecosystem
                return (iEcosystem != CommonEcosystem) && (iEcosystem < NbEcosystems);
        }
}


/*
 *
 */
struct TCraftPartInfo
{
        CSString                Name;
        CSString                Path;
        uint8                   PartIndex;
        bool                    Enabled;
};


/*
 *
 */
class CCraftParts
{
public:

        ///
        CCraftParts() : CraftParts( NbFaberElements )
        {
                for ( uint i=0; i!=NbFaberElements; ++i )
                {
                        CraftParts[i].PartIndex = i;
                        CraftParts[i].Enabled = false;
                }
        }

        ///
        void    registerPartChars( const CSString& parts )
        {
                for ( string::size_type p=0; p!=parts.size(); ++p )
                {
                        uint index = (uint)(parts[p] - 'A');
                        CraftParts[index].Enabled = true;
                }
        }

        ///
        bool    isEnabled( uint index ) const
        {
                return CraftParts[index].Enabled;
        }

        ///
        void    getNamesAndPaths( const vector<string>& paths )
        {
                uint i = 0;
                vector<string>::const_iterator ip;
                for ( ip=paths.begin(); ip!=paths.end(); ++ip )
                {
                        if ( i >= CraftParts.size() )
                                nlerror( "Mismatch between sitem DFN and constant (nb of craft parts)" );

                        CraftParts[i].Path = (*ip);
                        string::size_type p = (*ip).rfind( '.' ) + 1; // string::npos+1 gives 0
                        CraftParts[i].Name = (*ip).substr( p );
                        nldebug( "%u: %s", ip-paths.begin(), CraftParts[i].Name.c_str() );
                        ++i;
                }
        }

        ///
        TCraftPartInfo& operator[] ( uint index ) { return CraftParts[index]; }

        vector< TCraftPartInfo > CraftParts;

};

typedef map<CSString, TFamInfo, CUnsensitiveSStringLessPred > CFamMap;
CFamMap         FamSet;
CCraftParts CraftParts;

enum TFamAndPropLine {
        LFam, LGroup, LCraftParts, LCiv, LFreq, LLoc,
        LIconMain, LIconBk, LIconOv1, LIconOv2, LIconSpecial,
        LCraftPlans, LGenerateOnly,
        LBaseOfRemarkableStatIndices,
        LColorIndex = LBaseOfRemarkableStatIndices + NB_REMARKABLE_STAT_INDICES,
        LJewelProtIndex,
        NbFamAndPropCols };


// static
bool            TFamInfo::mustGenerateFamily( uint iFamily )
{
        if ( families[iFamily].empty() )
                return false;
        else if ( ! TFamInfo::UseGenerateOnly )
                return true;
        else
        {
                TFamInfo& famInfo = FamSet[families[iFamily]];
                return ( famInfo.GenerateOnly );
        }
}


/*
 *
 */
CSString        getShortFaberElemString( uint rFaberElem )
{
        string& longString = CraftParts[rFaberElem].Name;
        return reinterpret_cast<CSString&>(longString.substr( longString.find( "(" ) + 1 )).rightCrop( 1 );
}


/*
 *
 */
TCiv getCivFromStr( const CSString& civStr )
{
        if ( civStr.empty() )
                return AllCiv;
        else
        {
                for ( uint i=0; i!=NbCiv; ++i )
                {
                        if ( civStr == CSString(CivNames[i]) )
                                return (TCiv)i;
                }
                nlwarning( "Unknown civ '%s'", civStr.c_str() );
                return AllCiv;
        }
}


/*
 *
 */
uint getFreqFromStr( const CSString& freqStr )
{
        uint f = atoi( freqStr.c_str() );
        if ( (f < 1) && (f > 5) )
                nlwarning( "Unknown freq '%s'", freqStr.c_str() );
        return f;
}


/*
 * Returns ~0 if s is empty
 */
TGroup getNewOrExistingGroupFromStr( const CSString& s )
{
        uint i = getIndexFromString( s, groups, false );
        if ( i == ~0 )
        {
                if ( s.empty() )
                        return ~0;
                else
                {
                        i = groups.size();
                        groups.push_back( s );
                        nlinfo( "New group: %s (%u)", s.c_str(), i );
                }
        }
        return i;
}


/*
 *
 */
void    deliverFamAndProp( vs& line )
{
        if ( line.size() < NbFamAndPropCols )
                line.resize( NbFamAndPropCols );

        cleanExteriorWhitespace( line );

        if ( line[LFam].empty() )
        {
                // Load special icons
                if ( (line.size() >= LIconSpecial+1) && (! line[LIconSpecial].empty()) && (line[LIconMain].empty()) )
                {
                        /*if ( line.size() >= LIconMain+1 )
                                Icons[line[LIconSpecial]].Icon = line[LIconMain];*/
                        if ( line.size() >= LIconBk+1 )
                                Icons[line[LIconSpecial]].IconBackground = line[LIconBk];
                        if ( line.size() >= LIconOv1+1 )
                                Icons[line[LIconSpecial]].IconOver = line[LIconOv1];
                }
                return;
        }

        // Load icons of families
        if ( line.size() >= LIconMain+1 )
        {
                Icons[line[LFam]].Icon = line[LIconMain];
                if ( ! line[LGroup].empty() )
                {
                        // For group, set icon of first family of group found! (for forage source knowledge)
                        if ( Icons.find( line[LGroup] ) == Icons.end() )
                        {
                                Icons[line[LGroup]].Icon = line[LIconMain];
                        }
                }
        }
        if ( line.size() >= LIconBk+1 )
                Icons[line[LFam]].IconBackground = line[LIconBk];
        if ( line.size() >= LIconOv1+1 )
                Icons[line[LFam]].IconOver = line[LIconOv1];

        TFamInfo& famInfo = FamSet[line[LFam]];
        famInfo.IsActive = true;
        /*if ( ! line[LCraftParts].empty() )
        {
                // Store by property (line[LProp])
                famInfo.Properties.push_back( line[LProp] );
                famInfo.CraftPartsByProp.push_back( famInfo.CompatibleCraftParts.size() ); // beginning of craft parts chars
                famInfo.CompatibleCraftParts += line[LCraftParts];
                CraftParts.registerPartChars( line[LCraftParts] );
                famInfo.Civs.push_back( getCivFromStr( line[LCiv] ) );
                famInfo.Freqs.push_back( getFreqFromStr( line[LFreq] ) );
                famInfo.IsInDeposits = line[LLoc].contains( "D" );
                famInfo.IsInCreatures = line[LLoc].contains( "C" );
                if ( ! (famInfo.IsInDeposits || famInfo.IsInCreatures) )
                        nlwarning( "Unknown loc for %s", line[LFam].c_str() );
        }*/

        for ( string::size_type p=0; p!=line[LCraftParts].size(); ++p )
        {
                // Store by property = craft part (each char of line[LCraftParts])
                CSString craftPart = string( 1, line[LCraftParts][p]);
                famInfo.Properties.push_back( craftPart );
                famInfo.CraftPartsByProp.push_back( famInfo.CompatibleCraftParts.size() );
                famInfo.CompatibleCraftParts += craftPart;
                CraftParts.registerPartChars( craftPart );
                famInfo.Civs.push_back( getCivFromStr( line[LCiv] ) );
                famInfo.Freqs.push_back( getFreqFromStr( line[LFreq] ) );
        }
        if ( line[LCraftParts].empty() )
        {
                famInfo.Freqs.push_back( getFreqFromStr( line[LFreq] ) ); // freq needed for Rarity computation
        }
        famInfo.Group = getNewOrExistingGroupFromStr( line[LGroup] );
        famInfo.IsInDeposits = line[LLoc].contains( "D" );
        famInfo.IsInCreatures = line[LLoc].contains( "C" );
        if ( ! (famInfo.IsInDeposits || famInfo.IsInCreatures) )
        {
                famInfo.SpecialCreatureTag = line[LLoc];
                if ( (famInfo.SpecialCreatureTag[0] != 'G') && (famInfo.SpecialCreatureTag[0] != 'I') )
                        nlwarning( "Unknown loc %s for %s", line[LLoc].c_str(), line[LFam].c_str() );
        }
        famInfo.IsForMission = line[LCraftParts].empty();
        for ( uint i=0; i!=NB_REMARKABLE_STAT_INDICES; ++i )
        {
                if ( line[LBaseOfRemarkableStatIndices+i].empty() && (! line[LCraftParts].empty()) && (line[LFreq] != "0") )
                        nlerror( "%s has empty stat index %u", line[LFam].c_str(), i );
                famInfo.RemarkableStatIndex[i] = atoi( line[LBaseOfRemarkableStatIndices+i].c_str() );
        }
        if ( ! line[LColorIndex].empty() )
                famInfo.ColorIndex = atoi( line[LColorIndex].c_str() );
        if ( ! line[LJewelProtIndex].empty() )
                famInfo.JewelProtIndex = atoi( line[LJewelProtIndex].c_str() );
        bool markedForGeneration = (line[LGenerateOnly] == "X");
        if ( (!markedForGeneration) && famInfo.GenerateOnly )
        {
                nlwarning( "Found duplicate family line with different GenerateOnly setting" );
        }
        else
        {
                famInfo.GenerateOnly = markedForGeneration;
        }
        if ( famInfo.GenerateOnly )
                ++TFamInfo::UseGenerateOnly;
}


typedef map< TGroup, set<uint32> > CGroupMap;


/*
 *
 */
void    loadFamAndProp( const string& filename, bool displayAll )
{
        loadCSVFile( filename.c_str(), deliverFamAndProp );

        if ( displayAll )
        {
                set<CSString, CUnsensitiveSStringLessPred> propSet;
                CGroupMap groupMap;

                /// Generate contents of item_mp_family.typ (and fill group map)
                nlinfo( "item_mp_family.typ:" );
                InfoLog->displayRawNL( "<DEFINITION Label=\"Undefined\" Value=\"0\"/>" );
                uint i = 1;
                for ( CFamMap::iterator iss=FamSet.begin(); iss!=FamSet.end(); ++iss )
                {
                        const CSString& famStr = (*iss).first;
                        TFamInfo& famInfo = (*iss).second;
                        InfoLog->displayRawNL( "<DEFINITION Label=\"%s\" Value=\"%u\"/>", famStr.c_str(), i );

                        // Get info about props and group
                        for ( vs::iterator ip=famInfo.Properties.begin(); ip!=famInfo.Properties.end(); ++ip )
                        {
                                propSet.insert( *ip );
                        }
                        groupMap[ famInfo.Group ].insert( i ); // ~0 is for "no group" (creature's RMs only)
                        ++i;
                }

                /*
                /// Generate family-specialized forage search bricks (TODO)
                nlinfo( "Family-specialized forage search bricks:");
                i = 1;
                for ( CGroupMap::iterator igm=groupMap.begin(); igm!=groupMap.end(); ++igm )
                {
                        CSString skill = toString( "SHFM%u", i );
                        CSString rmgrpBrickCode = toString( "BHFPMB%02u", i );
                        CSString rmfamBrickFamCode = "BHFPMI" + string( 1, (char)'A' + ((char)(i-1)) );
                        uint j = 1;
                        for ( set<uint32>:::iterator ifs=(*igm).begin(); ifs!=(*igm).end(); ++ifs )
                        {
                                // TODO: modifier of modifier
                                CSString brickCode = rmfamBrickFamCode + toString( "%02u", j );
                                InfoLog->displayRawNL( "%s\t80\t%s\t%u\t\t%s\t\tFG_RMFAM_FILT: %u\t", brickCode.c_str(), rmgrpBrickCode.c_str(), j, skill.c_str(), (*ifs) );
                                ++j;
                        }
                }

                /// Generate family-specialized forage search phrases (TODO)
                nlinfo( "Family-specialized forage search phrases:");
                i = 1;
                for ( CFamMap::iterator iss=FamSet.begin(); iss!=FamSet.end(); ++iss )
                {
                        const CSString& famStr = (*iss).first;
                        TFamInfo& famInfo = (*iss).second;
                        InfoLog->displayRawNL( "<DEFINITION Label=\"%s\" Value=\"%u\"/>", famStr.c_str(), i );
                }

                /// Generate family-specialized forage extraction bricks (TODO)
                nlinfo( "Family-specialized forage extraction bricks:");
                i = 1;
                for ( CFamMap::iterator iss=FamSet.begin(); iss!=FamSet.end(); ++iss )
                {
                        const CSString& famStr = (*iss).first;
                        TFamInfo& famInfo = (*iss).second;
                        InfoLog->displayRawNL( "<DEFINITION Label=\"%s\" Value=\"%u\"/>", famStr.c_str(), i );
                        ++i;
                }

                /// Generate family-specialized forage extraction phrases (TODO)
                nlinfo( "Family-specialized forage extraction phrases:");
                i = 1;
                for ( CFamMap::iterator iss=FamSet.begin(); iss!=FamSet.end(); ++iss )
                {
                        const CSString& famStr = (*iss).first;
                        TFamInfo& famInfo = (*iss).second;
                        InfoLog->displayRawNL( "<DEFINITION Label=\"%s\" Value=\"%u\"/>", famStr.c_str(), i );
                        ++i;
                }*/

                /// Generate item_mp_property.typ
                nlinfo( "Item parts as props:" );
                InfoLog->displayRawNL( "<DEFINITION Label=\"Undefined\" Value=\"0\"/>" );
                i = 1;
                for ( set<CSString, CUnsensitiveSStringLessPred>::iterator iss=propSet.begin(); iss!=propSet.end(); ++iss )
                {
                        InfoLog->displayRawNL( "<DEFINITION Label=\"%s\" Value=\"%u\"/>", (*iss).c_str(), i );
                        ++i;
                }

                /// Generate item_mp_group.typ
                nlinfo( "Groups:" );
                InfoLog->displayRawNL( "<DEFINITION Label=\"Undefined\" Value=\"0\"/>" );
                i = 1;
                for ( CGroupMap::iterator igm=groupMap.begin(); igm!=groupMap.end(); ++igm )
                {
                        if ( (*igm).first != ~0 )
                        {
                                InfoLog->displayRawNL( "<DEFINITION Label=\"%s\" Value=\"%u\"/>", (groups[(*igm).first]).c_str(), i );
                                ++i;
                        }
                }

                /*
                /// Generate group-specialized forage search bricks (TODO)
                nlinfo( "Group-specialized forage search bricks:");
                i = 1;
                for ( CGroupMap::iterator igm=groupMap.begin(); igm!=groupMap.end(); ++igm )
                {
                        CSString skill = toString( "SHFM%u", i );
                        CSString rmgrpBrickCode = toString( "BHFPMB%02u", i );
                        ++i;
                }

                /// Generate group-specialized forage search phrases
                nlinfo( "Group-specialized forage search phrases:");
                i = 1;
                for ( set<CSString, CUnsensitiveSStringLessPred>::iterator iss=groupSet.begin(); iss!=groupSet.end(); ++iss )
                {
                        if ( (*iss).empty() )
                                continue;
                        InfoLog->displayRawNL( "<DEFINITION Label=\"%s\" Value=\"%u\"/>", (*iss).c_str(), i );
                        ++i;
                }

                /// Generate group-specialized forage extraction bricks
                nlinfo( "Group-specialized forage extraction bricks:");
                i = 1;
                for ( set<CSString, CUnsensitiveSStringLessPred>::iterator iss=groupSet.begin(); iss!=groupSet.end(); ++iss )
                {
                        if ( (*iss).empty() )
                                continue;
                        InfoLog->displayRawNL( "<DEFINITION Label=\"%s\" Value=\"%u\"/>", (*iss).c_str(), i );
                        ++i;
                }

                /// Generate group-specialized forage extraction phrases
                nlinfo( "Group-specialized forage extraction phrases:");
                i = 1;
                for ( set<CSString, CUnsensitiveSStringLessPred>::iterator iss=groupSet.begin(); iss!=groupSet.end(); ++iss )
                {
                        if ( (*iss).empty() )
                                continue;
                        InfoLog->displayRawNL( "<DEFINITION Label=\"%s\" Value=\"%u\"/>", (*iss).c_str(), i );
                        ++i;
                }*/

                nlinfo( "TODO: Keep old values when adding new entries" );
                nlinfo( "Don't forget to regen craft plans and to map localized texts" );
        }
}


/*
 * Multi-indexed array.
 * NC is the number of columns.
 */
template <uint32 NC>
class CSortableData
{
public:

        /// A row is made of fields, usually 1 per column but there may be more than one (each one is a key)
        struct TSortableItem
        {
                std::vector<std::string>                Fields [NC];

                ///
                void                    push( uint32 column, const std::string& f, bool allowDuplicates=false )
                {
                        if ( (allowDuplicates) || (find( Fields[column].begin(), Fields[column].end(), f ) == Fields[column].end()) )
                                Fields[column].push_back( f );
                }

                /**
                 * Display the item as a row of a HTML table.
                 * If (key!=previousKey) and (name==previousName), the row will not be displayed entirely to save space
                 *
                 * \param keyColumn If not ~0, column used for sorting => this column displays only the field matching the key
                 * \param key The key used for sorting (see keyColumn)
                 * \param previousKey Previous key
                 * \param nameColumn If not ~0, column used for the unique name (column must have exaclty one element)
                 * \param previousName Previous name
                 */
                std::string             toHTMLRow( uint32 keyColumn=~0, const string& key=string(), const string& previousKey=string(),
                                                                   uint32 nameColumn=~0, const string& previousName=string() ) const
                {
                        std::string s = "<tr>";
                        bool lightMode = (nameColumn == ~0) ? false : ((key != previousKey) && (Fields[nameColumn][0] == previousName));
                        for ( uint32 c=0; c!=NC; ++c )
                        {
                                s += "<td>";
                                if ( c == keyColumn )
                                        s += key; // key should be a substr of toString( c )
                                else
                                {
                                        if ( lightMode )
                                                s += "\"";
                                        else
                                                s += columnToString( c );
                                }
                                s += "</td>";
                        }
                        s += "</tr>\n";
                        return s;
                }


                ///
                std::string             toCSVLine( char columnSeparator=',', string internalSeparator=" - ", uint32 keyColumn=~0, const string& key=string(), const string& previousKey=string(),
                                                                   uint32 nameColumn=~0, const string& previousName=string()  ) const
                {
                        std::string s;
                        bool lightMode = (nameColumn == ~0) ? false : ((key != previousKey) && (Fields[nameColumn][0] == previousName));
                        for ( uint32 c=0; c!=NC; ++c )
                        {
                                if ( c == keyColumn )
                                        s += key; // key should be a substr of columnToString( c )
                                else
                                {
                                        if ( lightMode )
                                                s += "\"";
                                        else
                                                s += columnToString( c, internalSeparator );
                                }
                                s += columnSeparator;
                        }
                        s += "\n";
                        return s;
                }

                ///
                std::string             columnToString( uint32 column, const std::string& internalSeparator=", " ) const
                {
                        std::string s;
                        std::vector<std::string>::const_iterator ivs;
                        for ( ivs=Fields[column].begin(); ivs!=Fields[column].end(); ++ivs )
                        {
                                if ( ivs!=Fields[column].begin() )
                                        s += internalSeparator;
                                s += (*ivs);
                        }
                        return s;
                }
        };

        typedef std::multimap< std::string, uint32 > CLookup; // key to index (not pt because reallocation invalidates pointers)
        typedef std::vector< TSortableItem > CItems;

        /// Init
        void    init( bool enabled )
        {
                _Enabled = enabled;
        }

        /// Add a row
        void    addItem( const TSortableItem& item )
        {
                if ( ! _Enabled )
                        return;

                _Items.push_back( item );
                for ( uint32 c=0; c!=NC; ++c )
                {
                        for ( std::vector<std::string>::const_iterator ik=item.Fields[c].begin(); ik!=item.Fields[c].end(); ++ik )
                        {
                                _Indices[c].insert( make_pair( *ik, _Items.size()-1 ) );
                        }
                }
        }

        /**
         * Update a row (found by the first column, which must have exactly one element).
         * Returns true if it existed before, false if it's being created.
         * If it existed before:
         * - Does not remove elements that already exist and are not in the new item
         * - Adds the new elements found in the new item at the specified columns, and updates lookup map
         */
        bool    updateItemAppend( const TSortableItem& item, uint32 column )
        {
                if ( ! _Enabled )
                        return true; // quiet

                uint32 nameColumn = 0;
                CLookup::iterator ilk = _Indices[nameColumn].find( item.Fields[nameColumn][0] );
                if ( ilk != _Indices[nameColumn].end() )
                {
                        uint32& index = (*ilk).second;

                        // Update map for the specified column
                        // and update item column
                        for ( std::vector<std::string>::const_iterator ivs=item.Fields[column].begin(); ivs!=item.Fields[column].end(); ++ivs )
                        {
                                ilk = _Indices[column].find( *ivs );
                                if ( (ilk == _Indices[column].end()) || (  getRow( (*ilk).second ).Fields[nameColumn][0] != item.Fields[nameColumn][0]) )
                                {
                                        _Indices[column].insert( make_pair( *ivs, index ) );
                                        _Items[index].Fields[column].push_back( *ivs );
                                }
                        }

                        return true;
                }
                else
                {
                        addItem( item );
                        return false;
                }
        }

        /**
         * Update a row (found by the first column, which must have exactly one element).
         * Returns true if it existed before, false if it's being created.
         * If it existed before:
         * - Does not update lookup maps or item for columns that were already present.
         * - Adds entries in lookup maps and updates item for new columns (fields that were empty).
         */
        /*bool  updateItemAppend( const TSortableItem& item )
        {
                if ( ! _Enabled )
                        return true; // quiet

                CLookup::iterator ilk = _Indices[0].find( item.Fields[0][0] );
                if ( ilk != _Indices[0].end() )
                {
                        uint32& index = (*ilk).second;

                        for ( uint32 c=1; c!=NC; ++c )
                        {
                                // Update maps for previously empty columns
                                if ( _Items[index].Fields[c].empty() )
                                {
                                        for ( std::vector<std::string>::iterator ivs=item.Fields[c].begin(); ivs!=item.Fields[c].end(); ++ivs )
                                                _Indices[c].insert( make_pair( *ivs, index ) );
                                }

                                // Update item column
                                _Items[index].Fields[c] = item.Fields[c];
                        }

                        return true;
                }
                else
                {
                        addItem( item );
                        return false;
                }
        }*/

        /// Find or browse by key
        CLookup& lookup( uint32 column )
        {
                return _Indices[column];
        }

        /// Browse by adding order
        CItems& items()
        {
                return _Items;
        }

        /// Get a row by index
        TSortableItem&  getRow( uint32 index )
        {
                return _Items[index];
        }

private:
        
        CLookup                                                 _Indices [NC];

        CItems                                                  _Items;

        bool                                                    _Enabled;
};


typedef CSortableData<DtNbCols> CRMData;
typedef CRMData::TSortableItem TRMItem;


/**
 *
 */
class CProducedDocHtml
{
public:

        ///
        CProducedDocHtml() : _File(NULL), _Enabled(false) {}

        ///
        void    open( const std::string& filename, const std::string& title, bool enableFlag )
        {
                _Enabled = enableFlag;
                if ( ! _Enabled )
                        return;

                _File = NLMISC::nlfopen( filename, "wt" );
                fprintf( _File, ("<html><head>\n<meta http-equiv=\"content-type\" content=\"text/html; charset=UTF-8\">\n<title>" + title + "</title>\n</head><body>\n").c_str() );
        }

        ///
        void    write( const std::string& htmlCode )
        {
                if ( ! _Enabled )
                        return;

                fprintf( _File, htmlCode.c_str() );
        }

        ///
        void    writeln( const std::string& htmlCode )
        {
                write( htmlCode + "\n" );
        }

        ///
        void    writebln( const std::string& htmlCode )
        {
                write( htmlCode + "<br>\n" );
        }

        ///
        void    writepln( const std::string& htmlCode )
        {
                write( "<p>" + htmlCode + "</p>\n" );
        }

        ///
        void    save()
        {
                if ( ! _Enabled )
                        return;

                fprintf( _File, "</body></html>\n" );
                fclose( _File );
        }

private:

        FILE    *_File;
        bool    _Enabled;
};


/**
 *
 */
class CProducedDocCSV
{
public:

        ///
        CProducedDocCSV() : _File(NULL), _Enabled(false) {}

        ///
        void    open( const std::string& filename, bool enableFlag )
        {
                _Enabled = enableFlag;
                if ( ! _Enabled )
                        return;

                _File = NLMISC::nlfopen( filename, "wt" );
        }

        ///
        void    write( const std::string& data )
        {
                if ( ! _Enabled )
                        return;

                fprintf( _File, data.c_str() );
        }

        ///
        void    writeln( const std::string& data )
        {
                write( data + "\n" );
        }

        ///
        void    save()
        {
                if ( ! _Enabled )
                        return;

                fclose( _File );
        }

private:

        FILE    *_File;
        bool    _Enabled;
};



/**
 *
 */
class CGenRawMaterial
{
public:

        /// Constructor
        CGenRawMaterial( const std::string& sheetName = std::string() ) : SheetName(sheetName), ILocation(~0), IFamily(~0), IEcosystem(NbEcosystems), StatQuality(InvalidStatQuality), Color(InvalidColor), StatEnergyAvg(0)
        {}

        /// Serial
        void            serial( NLMISC::IStream& s )
        {
                s.serial( SheetName );
                s.serial( (uint32&)ILocation );
                s.serial( (uint32&)IFamily );
                s.serial( (uint32&)Group );
                s.serial( (uint32&)IEcosystem );
                s.serial( (uint32&)StatQuality );
                s.serial( (sint32&)Color );
                //s.serial( (uint32&)SapLoadLevel );
                //s.serial( (uint32&)Rarity );
                s.serial( (sint32&)StatEnergyAvg );
                s.serial( (uint32&)MaxLevel );
                s.serialCont( RMProperties );
                s.serialCont( IPropertyDepths );
                s.serialCont( RMCraftCharacs );
        }

        /// Computes randomly RMCraftCharacs, IPropertyDepths... Returns false if the RM must NOT be generated.
        bool    computeCraftCharacs( uint iVariant, const CSString& sheetName );

        ///
        void    writeSheet( CForm *form );

        ///
        void    loadSheet( CForm *form, const std::string& sheetName, bool full );

        ///
        void    collectStats( TRMItem& item, CMainStat& mainStats );

        /// Return average of energies (including max quality as half of the balance)
        /*float getEnergyAvg() const
        {
                if ( RMCraftCharacs.empty() )
                        return 0.0f;
                else
                {
                        float sum = 0.0f;
                        for ( list<CFaberCharacteristics>::const_iterator ics=RMCraftCharacs.begin(); ics!=RMCraftCharacs.end(); ++ics )
                        {
                                sum += (*ics).ActualEnergy;
                        }
                        //return (sum + (float)MaxQuality / 250.0f) / ((float)RMCraftCharacs.size() + 1);
                        return (sum / (float)RMCraftCharacs.size()); // now, MaxQuality is not part of the average
                }
        }*/

        ///
        float   getOriginalityAvg() const
        {
                float sum = 0.0f;
                for ( list<CFaberCharacteristics>::const_iterator ics=RMCraftCharacs.begin(); ics!=RMCraftCharacs.end(); ++ics )
                {
                        sum += (*ics).ActualOriginality;
                }
                return sum / (float)RMCraftCharacs.size();
        }

        ///
        float   getOriginalityMax() const
        {
                float maxOriginality = 0.0f;
                for ( list<CFaberCharacteristics>::const_iterator ics=RMCraftCharacs.begin(); ics!=RMCraftCharacs.end(); ++ics )
                {
                        if ( (*ics).ActualOriginality > maxOriginality )
                                maxOriginality = (*ics).ActualOriginality;
                }
                return maxOriginality;
        }

        ///
        void    fillPropertiesFromFamily()
        {
                vs& props = FamSet[familyStr()].Properties;
                RMProperties.resize( props.size() );
                for ( vs::iterator ip=props.begin(); ip!=props.end(); ++ip )
                {
                        RMProperties[ip-props.begin()] = getIndexFromString( *ip, properties );
                }
        }

        ///
        bool    hasCraftPart( uint craftPartIndex )
        {
                return CraftParts.isEnabled( craftPartIndex ) && (FamSet[familyStr()].CompatibleCraftParts.find( string( 1, (char)'A' + craftPartIndex ).c_str() ) != string::npos);
        }

        ///
        /*TCiv  getCivSpec( uint craftPartIndex, const TFamInfo& famInfo )
        {
                TCiv civ = NbCiv;       
                for ( vector<TCiv>::const_iterator ivc=famInfo.Civs.begin(); ivc!=famInfo.Civs.end(); ++ivc )
                {
                        // Skip those not matching the current rFaberElem
                        if ( famInfo.getCraftPartForProp( ivc-famInfo.Civs.begin() ).find( string( 1, (char)('A' + craftPartIndex) ).c_str() ) != string::npos )
                        {
                                if ( (civ != NbCiv) && ((*ivc) != civ) )
                                {
                                        nlwarning( "Different civ specializations for %s, %s (%s and %s)", familyStr().c_str(), getShortFaberElemString( craftPartIndex ).c_str(), CivNames[civ], CivNames[*ivc] );
                                        return AllCiv;
                                }
                                civ = (*ivc);
                        }
                }
                if ( civ == NbCiv )
                        return AllCiv;
                else if ( civ = 
                        return civ;
        }*/

        ///
        TCiv    getCivSpec( TEcosystem iEcosystem, TStatQuality statQuality )
        {
                if ( (statQuality <= Fine) || (iEcosystem >= NbEcosystems) )
                        return AllCiv;
                else
                        return EcosystemToCiv[iEcosystem];
        }

        ///
        /*TCiv  getMainCivSpec( const TFamInfo& famInfo )
        {
                TCiv civ = NbCiv;
                for ( list<CFaberCharacteristics>::const_iterator ics=RMCraftCharacs.begin(); ics!=RMCraftCharacs.end(); ++ics )
                {
                        TCiv civOfCraftPart = getCivSpec( (*ics).FaberElement, famInfo );
                        if ( (civ != NbCiv) && (civOfCraftPart != civ) )
                        {
                                return AllCiv;
                        }
                        civ = civOfCraftPart;
                }
                if ( civ == NbCiv )
                        return AllCiv;
                else
                        return civ;
        }*/

        ///
        /*const char *  getMainEcosystemSpec( const TFamInfo& famInfo )
        {
                return CivEcosystemCodes[getMainCivSpec( famInfo )];
        }*/

        /// Code
        CSString                                        SheetName;

        /// Index in locations
        uint32                                          ILocation;

        /// Index in families
        uint32                                          IFamily;

        ///
        CSString                                        familyStr() const { return families[IFamily]; }

        /// Group number
        TGroup                                          Group;

        ///
        CSString                                        groupStr() const { return Group==~0 ? "" : groups[Group]; }

        /// Index in ecosystems
        TEcosystem                                      IEcosystem;

        ///
        CSString                                        ecosystemStr() const { return ecosystems[IEcosystem]; }

        /// From Basic) to Supreme)
        TStatQuality                            StatQuality;

        /// From 'b' (Basic) to 'f' (Supreme)
        char                                            levelZoneLChar() const { return 'b' + (char)StatQuality; }

        /// Same
        void                                            setStatQuality( char levelZoneChar ) { StatQuality = (TStatQuality)(levelZoneChar - 'b'); }

        /// For creatures
        uint32                                          ILevelZone;

        /// Index in colors
        TColor                                          Color;

        ///
        CSString                                        colorStr() const { return colors[Color]; }

        /// Sap load level
        //uint32                                                SapLoadLevel;

        /// Rarity
        //uint32                                                Rarity;

        sint32                                          StatEnergyAvg;

        /// Max quality
        uint32                                          MaxLevel;

        /// Indices in properties
        vu                                                      RMProperties;

        ///
        CSString                                        propertyStr( uint32 p ) const { return properties[RMProperties[p]]; }

        ///
        vu                                                      IPropertyDepths;

        ///
        CSString                                        propertyDepthStr( uint32 p ) const { return PropertyDepths[IPropertyDepths[p]]; }

        ///
        CFaberCharacteristics           *getCraftSlot( uint rFaberElem )
        {
                std::list< CFaberCharacteristics >::iterator icl;
                for ( icl=RMCraftCharacs.begin(); icl!=RMCraftCharacs.end(); ++icl )
                {
                        if ( (*icl).FaberElement == rFaberElem )
                                return &(*icl);
                }
                return NULL;
        }

        /// Randomly generated characs
        std::list< CFaberCharacteristics >      RMCraftCharacs;
};


/**
 *
 */
class COriginalitySorter
{
public:

        typedef std::set< CGenRawMaterial* > CRMSet;
        typedef std::multimap< uint32, CGenRawMaterial*, std::greater<uint32> > CMultiMapByOriginality;

        ///
        COriginalitySorter() : RMByOriginalityByCraftSlot( NbFaberElements ) {}

        ///
        void    pushRM( CGenRawMaterial *rawMaterial )
        {
                RawMaterials.insert( rawMaterial );

                //InfoLog->displayRawNL( "Inserting RM" );
                std::list< CFaberCharacteristics >::const_iterator ilc;
                for ( ilc=rawMaterial->RMCraftCharacs.begin(); ilc!=rawMaterial->RMCraftCharacs.end(); ++ilc )
                {
                        //InfoLog->displayRawNL( "   %u: %s orig=%u", (*ilc).FaberElement, rawMaterial->SheetName.c_str(), (uint32)((*ilc).ActualOriginality*100.0f) );
                        RMByOriginalityByCraftSlot[(*ilc).FaberElement].insert( make_pair( (uint32)((*ilc).ActualOriginality*100.0f), rawMaterial ) );
                }
        }

        ///
        void    popAndDeleteRM( CGenRawMaterial *rawMaterial )
        {
                delete rawMaterial;
                RawMaterials.erase( rawMaterial );
        }

        ///
        bool    alreadyPopped( CGenRawMaterial *rawMaterial ) const
        {
                return RawMaterials.find( rawMaterial ) == RawMaterials.end();
        }

        /// fromPos and the returned iterator are the pos internal to the COriginalitySorter RM set
        /*CRMSet::const_iterator        getFirstRMNotInFamilyListFromPos( const set<uint>& familyList, TStatQuality statQuality, CRMSet::const_iterator fromPos ) const
        {
                CRMSet::const_iterator irm;
                for ( irm=fromPos; irm!=RawMaterials.end(); ++irm )
                {
                        if ( ((*irm)->StatQuality == statQuality) &&
                                 (familyList.find( (*irm)->IFamily ) == familyList.end()) )
                                return irm;
                }
                return RawMaterials.end();
        }*/

        ///
        CRMSet::iterator        getRMSetBegin() const { return RawMaterials.begin(); }

        ///
        CRMSet::iterator        getRMSetEnd() const { return RawMaterials.end(); }

        ///
        void    deleteAllRemainingRM()
        {
                CRMSet::iterator irm;
                for ( irm=RawMaterials.begin(); irm!=RawMaterials.end(); ++irm )
                {
                        delete (*irm);
                }
                RawMaterials.clear();
                // Does not clear the maps by originality
        }

        std::vector< CMultiMapByOriginality >   RMByOriginalityByCraftSlot;

private:

        CRMSet                                                                  RawMaterials;
};


#define checkColor( c ) nlassert( colors[c] == #c );


/*
 *
 */
void    loadDFNs( UFormLoader *formLoader )
{
        map<string, CDfnFieldInfo> dfnFields;
        NLMISC::CSmartPtr<UFormDfn> formDfn;
        formDfn = formLoader->loadFormDfn( (rmSheetType + ".dfn").c_str() );
        if ( ! formDfn )
                nlerror( "Can't find DFN for %s", rmSheetType.c_str() );
        vector<string> craftPartsPaths;
        fillFromDFN( formLoader, dfnFields, formDfn, "", rmSheetType, "mp.MpParam", craftPartsPaths );

        // Get craft parts
        CraftParts.getNamesAndPaths( craftPartsPaths );
        
        formDfn = formLoader->loadFormDfn( (crSheetType + ".dfn").c_str() );
        if ( ! formDfn )
                nlerror( "Can't find DFN for %s", crSheetType.c_str() );
        fillFromDFN( formLoader, dfnFields, formDfn, "", crSheetType );

        // Get lists of predefined values from sitem and creature DFN
        families = dfnFields["mp.Family"].TypePredefinedLabels;
        familyCodes = dfnFields["mp.Family"].TypePredefinedValues;
        groups = dfnFields["mp.Group"].TypePredefinedLabels;

        //properties = dfnFields["mp.Material property 1"].TypePredefinedLabels;
        //nlverify( removeEntryFromList( properties, "Undefined" ) != ~0 );
        properties.resize( craftPartsPaths.size() );
        for ( uint i=0; i!=properties.size(); ++i ) // now, use properties as item part list
                properties[i] = string( 1, (char)('A' + i) );

        ecosystems = dfnFields["mp.Ecosystem"].TypePredefinedLabels,
        colors = dfnFields["mp.MpColor"].TypePredefinedLabels,
        creatures = dfnFields["Basics.Race"].TypePredefinedLabels;
        seasons.push_back( "Winter" );
        seasons.push_back( "Spring" );
        seasons.push_back( "Summer" );
        seasons.push_back( "Autumn" );
        //removeEntryFromList( families, "Undefined" );
        //removeEntryFromList( familyCodes, "0" );
        nlverify( removeEntryFromList( ecosystems, "unknown" ) != ~0 );
        nlverify( removeEntryFromList( ecosystems, "Goo" ) != ~0 );
        nlassert( ecosystems[0] == "Common" );
        nlassert( ecosystems[1] == "Desert" ); // ensure we match with enum TEcosystem!
        nlassert( ecosystems[2] == "Forest" );
        nlassert( ecosystems[3] == "Lacustre" );
        nlassert( ecosystems[4] == "Jungle" );
        nlassert( ecosystems[5] == "PrimeRoots" );
        //removeEntryFromList( ecosystems, "Common" ); // TODO
        nlassert( NbEcosystems == ecosystems.size() );
        nlverify( removeEntryFromList( colors, "None" ) != ~0 );
        nlverify( removeEntryFromList( colors, "UserColor") != ~0 );
        nlassert( colors.size() == NbColors );
        checkColor( Red );
        checkColor( Beige );
        checkColor( Green );
        checkColor( Turquoise );
        checkColor( Blue );
        checkColor( Violet );
        checkColor( White );
        checkColor( Black );

        /*UndefinedProperty = getIndexFromString( "Undefined", properties );
        nlassert( UndefinedProperty != ~0 );*/
}


/*
 * Build RMFamilyIndicesByCreatureModel and DepositFamilyIndices
 */
void    dispatchFamiliesToLocations()
{
        for ( CFamMap::iterator iss=FamSet.begin(); iss!=FamSet.end(); ++iss )
        {
                const CSString& famStr = (*iss).first;
                TFamInfo& famInfo = (*iss).second;
                uint iFam = getIndexFromString( famStr, families );
                if ( famInfo.IsInDeposits )
                {
                        // Deposits
                        nlassert( iFam != ~0 );
                        DepositFamilyIndices.push_back( iFam );
                }
                if ( famInfo.IsInCreatures )
                {
                        // Extract creature name from left of family name (ASSUMES there's no blank in creature name)
                        CSString creaNameForRMFamily = famStr.splitTo( ' ' );

                        // Dispatch 
                        for ( CSkeletonMap::iterator icm=CreatureModels.begin(); icm!=CreatureModels.end(); ++icm )
                        {
                                const CSString& creaModel = (*icm).first;
                                TSkeletonInfo& modelInfo = (*icm).second;

                                if ( modelInfo.Name == creaNameForRMFamily )
                                {
                                        RMFamilyIndicesByCreatureModel[creaModel].push_back( iFam );
                                        //nlinfo( "+ %s for %s (now %u models registered)", famStr.c_str(), creaModel.c_str(), RMFamilyIndicesByCreatureModel.size() );
                                        modelInfo.IsUsed = true; // Name and AbbrevName are set by deliverCreatureModels()
                                }
                        }
                }
                else switch ( famInfo.SpecialCreatureTag[0] )
                {
                        // Goo & invasion/raid creatures
                        case 'G':
                        {
                                GooCreatureFamilyIndices.push_back( iFam );
                                nldebug( "Family %s selected for goo creatures", famStr.c_str() );
                                break;
                        }
                        case 'I':
                        {
                                if ( famInfo.SpecialCreatureTag.size() == 1 )
                                {
                                        InvasionRaidCreatureFamilyIndices['*'].push_back( iFam );
                                        nldebug( "Family %s selected for all invasion creatures", famStr.c_str() );
                                }
                                else
                                {
                                        for ( uint c=1; c!=famInfo.SpecialCreatureTag.size(); ++c )
                                        {
                                                InvasionRaidCreatureFamilyIndices[famInfo.SpecialCreatureTag[c]].push_back( iFam );
                                                nldebug( "Family %s selected for invasion creature of type %c", famStr.c_str(), famInfo.SpecialCreatureTag[c] );
                                        }
                                }
                                break;
                        }
                }
        }
}


/*
 * Returns the number of models used
 */
uint    checkSkeletons()
{
        uint32 nbSkeUsed = 0;
        for ( CSkeletonMap::const_iterator isc=CreatureModels.begin(); isc!=CreatureModels.end(); ++isc )
        {
                const string& skeFilename = (*isc).first;
                const TSkeletonInfo& ske = (*isc).second;
                const bool& used = (*isc).second.IsUsed;
                if ( used )
                {
                        nldebug( "Model %s (%s) %s", skeFilename.c_str(), ske.AbbrevName.c_str(), used?"used":"NOT USED" );
                        ++nbSkeUsed;
                }
                else
                        nlwarning( "Model %s %s", skeFilename.c_str(), used?"":"NOT USED" );
        }
        return nbSkeUsed;
}


/*
 *
 */
void    createDirectoryStructure()
{
        // Create the directory structure
        if ( WriteSheetsToDisk )
        {
                for ( uint32 i=0; i!=ecosystems.size(); ++i )
                {
                        string dirname = conventionalDirectory( ecosystems[i] );
                        if ( ! CFile::isExists( rawMaterialPath + dirname ) )
                        {
                                CFile::createDirectory( rawMaterialPath + dirname );
                        }
                        else
                        {
                                if ( ! CFile::isDirectory( rawMaterialPath + dirname ) )
                                {
                                        nlwarning( "%s already existing but not a directory!", (rawMaterialPath + dirname).c_str() );
                                }
                        }
                }
                string dirname = "_parent";
                if ( ! CFile::isExists( rawMaterialPath + dirname ) )
                {
                        CFile::createDirectory( rawMaterialPath + dirname );
                }
                else
                {
                        if ( ! CFile::isDirectory( rawMaterialPath + dirname ) )
                        {
                                nlwarning( "%s already existing but not a directory!", (rawMaterialPath + dirname).c_str() );
                        }
                }
        }
}


#endif // NL_SRG_UTILITIES_H

/* End of srg_utilities.h */