///////////////////////////////////////////////////////////////////////////////
//                                                         
// CegoQueryCache.cc
// -----------------
// Cego query cache implementation
//      
// Design and Implementation by Bjoern Lemke
//     
// (C)opyright 2000-2025 Bjoern Lemke
//
// IMPLEMENTATION MODULE
//
// Class: CegoQueryCache
// 
// Description: Query Cache Management
//
// Status: CLEAN
//
///////////////////////////////////////////////////////////////////////////////

#include <lfcbase/ThreadLock.h>
#include <lfcbase/Sleeper.h>
#include "CegoQueryCache.h"
#include "CegoXMLdef.h"
#include "CegoDefs.h"

#include <stdlib.h>
#include <string.h>

static ThreadLock _cacheLock[TABMNG_MAXTABSET];
extern bool __lockStatOn;

CegoQueryCache::QueryCacheEntry::QueryCacheEntry()
{
    _pCacheArray = 0;
    _numHit=0;
    _numUsed=0;
    _numRow = 0;
    _numCol = 0;
}

CegoQueryCache::QueryCacheEntry::QueryCacheEntry(const Chain& queryId)
{
    _queryId = queryId;
    _pCacheArray = 0;
    _numHit=0;
    _numUsed=0;
    _numRow = 0;
    _numCol = 0;
}

CegoQueryCache::QueryCacheEntry::QueryCacheEntry(const Chain& queryId, const SetT<CegoObject>& objList, ListT< ListT<CegoFieldValue> >* pCacheList, const ListT<CegoField>& cacheSchema)
{
    _queryId = queryId;    
    _objList = objList;

    // the cache entry has to be converted from list to array to be thread safe
    
    _pCacheArray = new CegoFieldValue**[ pCacheList->Size() ];
    ListT<CegoFieldValue>* pFVL = pCacheList->First();
    _numRow = 0;
    _numCol = 0;
    while ( pFVL )
    {
	CegoFieldValue** pCFVL = new CegoFieldValue*[pFVL->Size()];
	CegoFieldValue* pFV = pFVL->First();
	unsigned i = 0;
	while ( pFV )
	{
	    CegoFieldValue* pCFV = new CegoFieldValue(pFV->getLocalCopy());
	    pCFVL[i] = pCFV;
	    pFV = pFVL->Next();
	    i++;
	}
	_numCol = i;
       
	_pCacheArray[_numRow] = pCFVL;
	_numRow++;
	pFVL = pCacheList->Next();	
    }
    
    _cacheSchema = cacheSchema;
    _numHit=1;
    _numUsed=0;

    calcSize();
}

CegoQueryCache::QueryCacheEntry::~QueryCacheEntry()
{
}

bool CegoQueryCache::QueryCacheEntry::cleanCache()
{
    if ( _numUsed > 0 )
    {
	// cout << "Cache Id " << _queryId << " not clean" << endl;
	return false;
    }
    
    for ( unsigned i=0; i<_numRow; i++ )
    {
	for ( unsigned j=0; j<_numCol; j++ )
	    delete _pCacheArray[i][j];
	delete _pCacheArray[i];
    }
    delete _pCacheArray;

    _objList.Empty();
    _cacheSchema.Empty();
    
    _pCacheArray = 0;
    _numRow = 0;
    _numCol = 0;
    
    return true;
}

void CegoQueryCache::QueryCacheEntry::calcSize()
{    
    _size =  _queryId.length();
    CegoObject *pO = _objList.First();
    while ( pO )
    {
	_size += pO->size();
	pO = _objList.Next();
    }

    CegoField *pF = _cacheSchema.First();
    while ( pF )
    {
	_size += pF->size();
	pF = _cacheSchema.Next();
    }

    for ( unsigned i=0; i<_numRow; i++ )
    {
	for ( unsigned j=0; j<_numCol; j++ )
	{
	    CegoFieldValue *pF = _pCacheArray[i][j];
	    _size += pF->size();
	}
    }
}

unsigned CegoQueryCache::QueryCacheEntry::getSize() const
{
    return _size;
}

unsigned CegoQueryCache::QueryCacheEntry::getHashPos(unsigned hashSize) const
{
    return _queryId.getHashPos(hashSize);
}

const Chain CegoQueryCache::QueryCacheEntry::getQueryId() const
{
    return _queryId;
}

unsigned CegoQueryCache::QueryCacheEntry::getNumRows() const
{
    return _numRow;
}

unsigned CegoQueryCache::QueryCacheEntry::getNumUsed() const
{
    return _numUsed;
}

const SetT<CegoObject>& CegoQueryCache::QueryCacheEntry::getObjectList() const
{
    return _objList; 
}

CegoFieldValue*** CegoQueryCache::QueryCacheEntry::claimCache()
{
    _numUsed++;
    return _pCacheArray;
}

void CegoQueryCache::QueryCacheEntry::releaseCache()
{
    _numUsed--;
}

const ListT<CegoField>& CegoQueryCache::QueryCacheEntry::getSchema() const
{
    return _cacheSchema;
}

unsigned long CegoQueryCache::QueryCacheEntry::getHit() const
{
    return _numHit;
}

void CegoQueryCache::QueryCacheEntry::incHit()
{
    _numHit++;
}

CegoQueryCache::QueryCacheEntry& CegoQueryCache::QueryCacheEntry::operator = ( const CegoQueryCache::QueryCacheEntry& qce)
{
    _queryId = qce._queryId;
    _objList = qce._objList;
    _pCacheArray = qce._pCacheArray;
    _cacheSchema = qce._cacheSchema;
    _numHit = qce._numHit;
    _numRow = qce._numRow;
    _numCol = qce._numCol;
    return (*this);
}

bool CegoQueryCache::QueryCacheEntry::operator == ( const CegoQueryCache::QueryCacheEntry& qce)
{
    if ( _queryId == qce._queryId )
	return true;
    return false;
}

/////////////////////////
// Query Cache Methods //
/////////////////////////

CegoQueryCache::CegoQueryCache(unsigned tabSetId, unsigned maxEntry, unsigned maxSize, unsigned threshold, unsigned hashRange)
{
    _cacheLock[tabSetId].init(LCKMNG_LOCKWAITDELAY, __lockStatOn);
    _cacheLock[tabSetId].setId( Chain("QCACHE") + Chain("-") + Chain(tabSetId));

    _tabSetId = tabSetId;
    _maxEntry = maxEntry;
    _maxSize = maxSize;
    _threshold = threshold;
    _hashRange = hashRange;

    // cout << "Allocating Query Cache with MaxEntry=" << _maxEntry << " MaxSize=" << _maxSize << " HashRange=" << _hashRange << endl;
    _pQueryCache = new HashT<QueryCacheEntry>(_maxEntry, _hashRange);
    _usedSize = 0;
    _numFail = 0;

    _numTry = 0;
    _numHit = 0;    
}

CegoQueryCache::~CegoQueryCache()
{
    clean();
    delete _pQueryCache;
}

void CegoQueryCache::PR()
{
    _cacheLock[_tabSetId].readLock(DBM_LOCKTIMEOUT);
}

void CegoQueryCache::PW()
{
    _cacheLock[_tabSetId].writeLock(DBM_LOCKTIMEOUT);
}

void CegoQueryCache::V()
{
    _cacheLock[_tabSetId].unlock();
}

Element* CegoQueryCache::getCacheInfo()
{
    Element* pCacheInfo = new Element(XML_CACHEINFO_ELEMENT);

    Element *pN = new Element(XML_CACHE_ELEMENT);
    pN->setAttribute(XML_ATTRNAME_ATTR, Chain("MaxEntry"));
    pN->setAttribute(XML_VALUE_ATTR, Chain(_maxEntry));
    pCacheInfo->addContent(pN);

    pN = new Element(XML_CACHE_ELEMENT);
    pN->setAttribute(XML_ATTRNAME_ATTR, Chain("MaxSize"));
    pN->setAttribute(XML_VALUE_ATTR, Chain(_maxSize));
    pCacheInfo->addContent(pN);

    pN = new Element(XML_CACHE_ELEMENT);
    pN->setAttribute(XML_ATTRNAME_ATTR, Chain("Threshold"));
    pN->setAttribute(XML_VALUE_ATTR, Chain(_threshold));
    pCacheInfo->addContent(pN);
    
    pN = new Element(XML_CACHE_ELEMENT);
    pN->setAttribute(XML_ATTRNAME_ATTR, Chain("HashRange"));
    pN->setAttribute(XML_VALUE_ATTR, Chain(_hashRange));
    pCacheInfo->addContent(pN);
    
    pN = new Element(XML_CACHE_ELEMENT);
    pN->setAttribute(XML_ATTRNAME_ATTR, Chain("UsedSize"));
    pN->setAttribute(XML_VALUE_ATTR, Chain(_usedSize));
    pCacheInfo->addContent(pN);
    
    pN = new Element(XML_CACHE_ELEMENT);
    pN->setAttribute(XML_ATTRNAME_ATTR, Chain("HitRate"));

    double hitRate = 0.0;
    if ( _numTry > 0 )
	hitRate = 100.0 * ( (double)_numHit ) / ( (double)_numTry ); 
    Chain hrFormated = Chain(hitRate, "%3.2f") + Chain("%");

    pN->setAttribute(XML_VALUE_ATTR, hrFormated);
    pCacheInfo->addContent(pN);

    unsigned numEntry = _pQueryCache->numEntry();
    pN = new Element(XML_CACHE_ELEMENT);
    pN->setAttribute(XML_ATTRNAME_ATTR, Chain("NumEntry"));
    pN->setAttribute(XML_VALUE_ATTR, Chain(numEntry));
    pCacheInfo->addContent(pN);

    pN = new Element(XML_CACHE_ELEMENT);
    pN->setAttribute(XML_ATTRNAME_ATTR, Chain("NumFail"));
    pN->setAttribute(XML_VALUE_ATTR, Chain(_numFail));
    pCacheInfo->addContent(pN);

    return pCacheInfo;
}

Element* CegoQueryCache::getCacheList()
{
    Element* pCacheInfo = new Element(XML_CACHEINFO_ELEMENT);
    PR();

    try
    {
	QueryCacheEntry *pCE = _pQueryCache->First();
	while ( pCE )
	{
	    Element *pN = new Element(XML_CACHE_ELEMENT);
	    pN->setAttribute(XML_POS_ATTR, _pQueryCache->getPos());
	    pN->setAttribute(XML_ID_ATTR, pCE->getQueryId());
	    pN->setAttribute(XML_NUMROWS_ATTR, Chain(pCE->getNumRows()));
	    pN->setAttribute(XML_NUMHITS_ATTR, Chain(pCE->getHit()));
	    pN->setAttribute(XML_SIZE_ATTR, Chain(pCE->getSize()));
	    pCacheInfo->addContent(pN);
	    pCE = _pQueryCache->Next();
	}
    }
    catch ( Exception e )
    {
	V();
	throw Exception(EXLOC, Chain("Cannot get query cache info"), e);
    }
    V();
    return pCacheInfo;
}

unsigned CegoQueryCache::getMaxEntry() const
{
    return _maxEntry;
}

unsigned CegoQueryCache::getMaxSize() const
{
    return _maxSize;
}

unsigned CegoQueryCache::getThreshold() const
{
    return _threshold;
}

unsigned CegoQueryCache::getNumQueryCache() const
{
    return _pQueryCache->numEntry();
}

unsigned long long CegoQueryCache::getUsedSize() const
{
    return _usedSize;
}

unsigned CegoQueryCache::getNumFail() const
{
    return _numFail;
}

void CegoQueryCache::invalidate(const CegoObject& obj)
{
    bool isClean=false;

    unsigned tryDelay = QUERYCACHE_TRYDELAYBASE;
    unsigned tryCount = 0;
    
    while ( isClean == false && tryCount < QUERYCACHE_MAXTRY)
    {
	tryCount++;
	
	PW();

	try
	{
	    // reset to clean
	    isClean=true;
	    
	    QueryCacheEntry *pQCE = _pQueryCache->First();
	    while ( pQCE )
	    {
		if ( pQCE->getObjectList().Find(obj) )
		{
		    unsigned s = pQCE->getSize();
		    
		    if ( pQCE->cleanCache() )
		    {
			// cout << "Cache invalition, decreased size from " << _usedSize << " to " << _usedSize - s  << "(" << s << ")" << endl; 
			_usedSize = _usedSize - s;
			
			unsigned pos = _pQueryCache->getPos();
			if ( _pQueryCache->RemovePos(pos) == false )
			{
			    Chain msg = Chain("Cannot remove query cache entry ") + Chain(pQCE->getQueryId());
			    throw Exception(EXLOC, msg);
			}
		    
			pQCE = _pQueryCache->First();		    		    
		    }
		    else
		    {
			// was not able to cleanup, cache still in use
			// cout << "Cache still in use .." << endl;
			isClean=false;
			// we skip this entry and go to next
			
			pQCE = _pQueryCache->Next();
		    }
		}
		else
		{
		    // cache entry not relevant for object, go to next
		    pQCE = _pQueryCache->Next();
		}
	    }
	}
	catch ( Exception e )
	{
	    V();
	    Chain msg = Chain("Cannot invalidate query cache for object ") + obj.getName();
	    throw Exception(EXLOC, msg, e);			    
	}
	V();

	if ( isClean == false )
	{
	    Sleeper::milliSleep(tryDelay);
	    tryDelay = tryDelay * 2;
	}
    }

    if ( isClean == false )
    {
	Chain msg = Chain("Cannot invalidate query cache for object ") + obj.getName() + Chain(", too many tries");
	throw Exception(EXLOC, msg);			    
    }
}

CegoFieldValue*** CegoQueryCache::claimEntry(const Chain& queryId, ListT<CegoField>& cacheSchema, unsigned& cacheRows)
{
    CegoFieldValue*** pCE = 0;

    try
    {
	PR();
    }
    catch ( Exception e )
    {
	Chain msg = Chain("Cannot claim entry for queryid = ") + queryId;
	throw Exception(EXLOC, msg, e);			    
    }

    try
    {
	_numTry++;
	
	QueryCacheEntry *pQCE = _pQueryCache->Find( QueryCacheEntry(queryId) );
	
	if ( pQCE )
	{
	    pQCE->incHit();
	    pCE = pQCE->claimCache();
	    cacheSchema = pQCE->getSchema();
	    cacheRows = pQCE->getNumRows();
	    _numHit++;
	}
    }
    catch ( Exception e )
    {
	V();
	throw Exception(EXLOC, Chain("Cannot claim query cache entry"), e);
    }
    V();
    
    return pCE;
}

void CegoQueryCache::releaseEntry(const Chain& queryId)
{
    try
    {
	PR();
    }
    catch ( Exception e )
    {
	Chain msg = Chain("Cannot release entry for queryid = ") + queryId;
	throw Exception(EXLOC, msg, e);			    
    }
    
    try
    {
	QueryCacheEntry *pQCE = _pQueryCache->Find( QueryCacheEntry(queryId) );
	
	if ( pQCE )
	{
	    pQCE->releaseCache();
	}
    }
    catch ( Exception e )
    {
	V();
	Chain msg = Chain("Cannot release entry for queryid = ") + queryId;
	throw Exception(EXLOC, msg, e);			    
    }
    
    V();
}

bool CegoQueryCache::addEntry(const Chain& queryId, const SetT<CegoObject>& objectList, ListT< ListT<CegoFieldValue> >* pCacheList, const ListT<CegoField>& cacheSchema )
{       
    bool wasInserted = false;

    unsigned tryDelay = QUERYCACHE_TRYDELAYBASE;
    unsigned tryCount = 0;
    
    while ( wasInserted == false && tryCount < QUERYCACHE_MAXTRY)
    {
	tryCount++;

	try
	{	    
	    PW();	   
	}
	catch ( Exception e )
	{
	    Chain msg = Chain("Cannot add entry for queryid = ") + queryId;
	    throw Exception(EXLOC, msg, e);			    
	}

	try
	{	    
	    QueryCacheEntry *pQCE = _pQueryCache->Find( QueryCacheEntry(queryId));

	    // entry already exists ?
	    if ( pQCE )
	    {		
		V();
		return true;
	    }
	    
	    QueryCacheEntry qce(queryId, objectList, pCacheList, cacheSchema);

	    wasInserted = _pQueryCache->Insert(qce);
		
	    if ( wasInserted == false )
	    {
		QueryCacheEntry *pRE = 0;
		
		unsigned pos;
		unsigned long minHit = 0;
		QueryCacheEntry *pQCE = _pQueryCache->FirstInRange(qce);
		while ( pQCE )
		{
		    if ( pQCE->getNumUsed() == 0 && ( minHit == 0 || pQCE->getHit() < minHit ) )
		    {			
			pos = _pQueryCache->getRangePos();
			pRE = pQCE;
			minHit = pQCE->getHit();
		    }
		    
		    pQCE = _pQueryCache->NextInRange();
		}
		
		if ( pRE )
		{
		    if ( pRE->cleanCache() )
		    {
			unsigned s = pRE->getSize();
			_usedSize = _usedSize - s;
			// cout << "Removing cache entry .." << endl;
			
			if ( _pQueryCache->RemovePos(pos) == false )
			{
			    Chain msg = Chain("Cannot remove query cache entry ") + Chain(pRE->getQueryId());
			    throw Exception(EXLOC, msg);
			}			
		    }
		}
		else
		{
		    // No appropriate entry found, we try again, if tryCount < MAXTRY
		    
		    // Chain msg = Chain("Cannot find appropriate query cache slot");
		    // throw Exception(EXLOC, msg);
		}
	    }
	    else
	    {
		_usedSize = _usedSize + qce.getSize();
	    }
	}
	catch ( Exception e )
	{
	    V();
	    throw Exception(EXLOC, Chain("Cannot add querycache entry"), e);
	}

	V();

	if ( wasInserted == false )
	{
	    Sleeper::milliSleep(tryDelay);
	    tryDelay = tryDelay * 2;
	}
    }
        
    if ( wasInserted == false )
	_numFail++;

    return wasInserted;
}

void CegoQueryCache::clean()
{
    bool isClean=false;
    
    while ( isClean == false )
    {
	isClean = true;
	
	try
	{	    
	    PW();	   
	}
	catch ( Exception e )
	{
	    Chain msg = Chain("Cannot clean querycache");
	    throw Exception(EXLOC, msg, e);			    
	}
	
	try
	{
	    QueryCacheEntry *pQCE = _pQueryCache->First();
	    while ( pQCE )
	    {	    	    
		// perhaps we have to wait for cache users,
		// so we repeat until cache is clean
		if ( pQCE->cleanCache() )
		{
		    unsigned pos = _pQueryCache->getPos();
		    if ( _pQueryCache->RemovePos(pos) == false )
		    {
			Chain msg = Chain("Cannot remove query cache entry ") + Chain(pQCE->getQueryId());
			throw Exception(EXLOC, msg);
		    }
		    
		    pQCE = _pQueryCache->First();
		}
		else
		{
		    // cout << "Cache not clean .." << endl;
		    isClean=false;
		    pQCE = _pQueryCache->Next();
		}
	    }
	}
	catch ( Exception e )
	{
	    V();

	    Chain msg = Chain("Cannot clean querycache");
	    throw Exception(EXLOC, msg, e);
	}
	V();	
    }

    _usedSize = 0;
}

void CegoQueryCache::getQCLockStat(unsigned& lockCount, unsigned long long &numRdLock, unsigned long long &numWrLock, unsigned long long &sumRdDelay, unsigned long long &sumWrDelay)
{
    lockCount = _cacheLock[_tabSetId].numLockTry();
    
    numRdLock = _cacheLock[_tabSetId].numReadLock();
    numWrLock = _cacheLock[_tabSetId].numWriteLock();
    sumRdDelay = 0;
    sumWrDelay = 0;

    if ( _cacheLock[_tabSetId].numReadLock() > 0 )
	sumRdDelay = _cacheLock[_tabSetId].sumReadDelay() / LCKMNG_DELRES;
    if ( _cacheLock[_tabSetId].numWriteLock() > 0 )
	sumWrDelay = _cacheLock[_tabSetId].sumWriteDelay() / LCKMNG_DELRES;
}

void CegoQueryCache::verifyLock(CegoAdminHandler* pAH)
{
    unsigned timeout = 3000;
    
    Chain info;
    
    info = Chain("QueryCache Lock TS ") + Chain(_tabSetId) + Chain(" - Read ...");
    if ( pAH )
	pAH->sendInfo(info);
	
    _cacheLock[_tabSetId].readLock(timeout);
    _cacheLock[_tabSetId].unlock();

    if ( pAH )
	pAH->sendInfo(Chain("ok\n"));

    info = Chain("QueryCache Lock TS ") + Chain(_tabSetId) + Chain(" - Write ...");
    if ( pAH )
	pAH->sendInfo(info);
	
    _cacheLock[_tabSetId].writeLock(timeout);
    _cacheLock[_tabSetId].unlock();
    
    if ( pAH )
	pAH->sendInfo(Chain("ok\n"));
    
}
