Documentation TYPO3 par Ameos

Disponible dans d'autres versions de TYPO3 -> 3.7.0  3.8.1  4.0.0  4.0.1  4.1.0   <- Available in other TYPO3 versions

class.indexer.php

<?php
/***************************************************************
*  Copyright notice
*
*  (c) 2001-2005 Kasper Skaarhoj (kasperYYYY@typo3.com)
*  All rights reserved
*
*  This script is part of the TYPO3 project. The TYPO3 project is
*  free software; you can redistribute it and/or modify
*  it under the terms of the GNU General Public License as published by
*  the Free Software Foundation; either version 2 of the License, or
*  (at your option) any later version.
*
*  The GNU General Public License can be found at
*  http://www.gnu.org/copyleft/gpl.html.
*  A copy is found in the textfile GPL.txt and important notices to the license
*  from the author is found in LICENSE.txt distributed with these scripts.
*
*
*  This script is distributed in the hope that it will be useful,
*  but WITHOUT ANY WARRANTY; without even the implied warranty of
*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
*  GNU General Public License for more details.
*
*  This copyright notice MUST APPEAR in all copies of the script!
***************************************************************/
require_once(PATH_t3lib.'class.t3lib_parsehtml.php');


class tx_indexedsearch_indexer {

                // Messages:
        var $reasons = array(
                -1 => 'mtime matched the document, so no changes detected and no content updated',
                -2 => 'The minimum age was not exceeded',
                1 => "The configured max-age was exceeded for the document and thus it's indexed.",
                2 => 'The minimum age was exceed and mtime was set and the mtime was different, so the page was indexed.',
                3 => 'The minimum age was exceed, but mtime was not set, so the page was indexed.',
                4 => 'Page has never been indexed (is not represented in the index_phash table).'
        );

                // HTML code blocks to exclude from indexing:
        var $excludeSections = 'script,style';

                // Supported Extensions for external files:
        var $external_parsers = array();                // External parser objects, keys are file extension names. Values are objects with certain methods.

                // Fe-group list (pages might be indexed separately for each usergroup combination to support search in access limited pages!)
        var $defaultGrList = '0,-1';

                // Min/Max times:
        var $tstamp_maxAge = 0;         // If set, this tells a number of seconds that is the maximum age of an indexed document. Regardless of mtime the document will be re-indexed if this limit is exceeded.
        var $tstamp_minAge = 0;         // If set, this tells a minimum limit before a document can be indexed again. This is regardless of mtime.
        var $maxExternalFiles = 0;      // Max number of external files to index.

        var $forceIndexing = FALSE;             // If true, indexing is forced despite of hashes etc.
        var $crawlerActive = FALSE;             // Set when crawler is detected (internal)

                // INTERNALS:
        var $defaultContentArray=array(
                'title' => '',
                'description' => '',
                'keywords' => '',
                'body' => '',
        );
        var $wordcount = 0;
        var $externalFileCounter = 0;

        var $conf = array();            // Configuration set internally (see init functions for required keys and their meaning)
        var $indexerConfig = array();   // Indexer configuration
        var $hash = array();            // Hash array, contains phash and phash_grouping
        var $file_phash_arr = array();  // Hash array for files
        var $contentParts = array();    // Content of TYPO3 page
        var $content_md5h = '';
        var $internal_log = array();    // Internal log
        var $indexExternalUrl_content = '';

        var $cHashParams = array();     // cHashparams array

        var $freqRange = 32000;
        var $freqMax = 0.1;

                // Objects:
        var $csObj;                             // Charset class object , t3lib_cs
        var $metaphoneObj;              // Metaphone object, if any
        var $lexerObj;                  // Lexer object for word splitting



        function hook_indexContent(&$pObj)      {

                        // Indexer configuration from Extension Manager interface:
                $indexerConfig = unserialize($GLOBALS['TYPO3_CONF_VARS']['EXT']['extConf']['indexed_search']);

                        // Crawler activation:
                        // Requirements are that the crawler is loaded, a crawler session is running and re-indexing requested as processing instruction:
                if (t3lib_extMgm::isLoaded('crawler')
                                && $pObj->applicationData['tx_crawler']['running']
                                && in_array('tx_indexedsearch_reindex', $pObj->applicationData['tx_crawler']['parameters']['procInstructions']))        {

                                // Setting simple log message:
                        $pObj->applicationData['tx_crawler']['log'][] = 'Forced Re-indexing enabled';

                                // Setting variables:
                        $this->crawlerActive = TRUE;    // Crawler active flag
                        $this->forceIndexing = TRUE;    // Force indexing despite timestamps etc.
                }

                        // Determine if page should be indexed, and if so, configure and initialize indexer
                if ($pObj->config['config']['index_enable'])    {
                        $this->log_push('Index page','');

                        if (!$indexerConfig['disableFrontendIndexing'] || $this->crawlerActive) {
                                if (!$pObj->page['no_search'])  {
                                        if (!$pObj->no_cache)   {

                                                        // Setting up internal configuration from config array:
                                                $this->conf = array();

                                                        // Information about page for which the indexing takes place
                                                $this->conf['id'] = $pObj->id;                          // Page id
                                                $this->conf['type'] = $pObj->type;                      // Page type
                                                $this->conf['sys_language_uid'] = $pObj->sys_language_uid;      // sys_language UID of the language of the indexing.
                                                $this->conf['MP'] = $pObj->MP;                          // MP variable, if any (Mount Points)
                                                $this->conf['gr_list'] = $pObj->gr_list;        // Group list

                                                $this->conf['cHash'] = $pObj->cHash;                                    // cHash string for additional parameters
                                                $this->conf['cHash_array'] = $pObj->cHash_array;                // Array of the additional parameters

                                                $this->conf['crdate'] = $pObj->page['crdate'];                  // The creation date of the TYPO3 page
                                                $this->conf['page_cache_reg1'] = $pObj->page_cache_reg1;        // reg1 of the caching table. Not known what practical use this has.

                                                        // Root line uids
                                                $this->conf['rootline_uids'] = array();
                                                foreach($pObj->config['rootLine'] as $rlkey => $rldat)  {
                                                        $this->conf['rootline_uids'][$rlkey] = $rldat['uid'];
                                                }

                                                        // Content of page:
                                                $this->conf['content'] = $pObj->content;                                        // Content string (HTML of TYPO3 page)
                                                $this->conf['indexedDocTitle'] = $pObj->convOutputCharset($pObj->indexedDocTitle);      // Alternative title for indexing
                                                $this->conf['metaCharset'] = $pObj->metaCharset;                        // Character set of content (will be converted to utf-8 during indexing)
                                                $this->conf['mtime'] = $pObj->register['SYS_LASTCHANGED'];      // Most recent modification time (seconds) of the content on the page. Used to evaluate whether it should be re-indexed.

                                                        // Configuration of behavior:
                                                $this->conf['index_externals'] = $pObj->config['config']['index_externals'];    // Whether to index external documents like PDF, DOC etc. (if possible)
                                                $this->conf['index_descrLgd'] = $pObj->config['config']['index_descrLgd'];              // Length of description text (max 250, default 200)

                                                        // Set to zero:
                                                $this->conf['recordUid'] = 0;
                                                $this->conf['freeIndexUid'] = 0;

                                                        // Init and start indexing:
                                                $this->init();
                                                $this->indexTypo3PageContent();

                                        } else $this->log_setTSlogMessage('Index page? No, page was set to "no_cache" and so cannot be indexed.');
                                } else $this->log_setTSlogMessage('Index page? No, The "No Search" flag has been set in the page header!');
                        } else $this->log_setTSlogMessage('Index page? No, Ordinary Frontend indexing during rendering is disabled.');
                }
                $this->log_pull();
        }








        /****************************
         *
         * Backend API
         *
         ****************************/

        function backend_initIndexer($id, $type, $sys_language_uid, $MP, $uidRL, $cHash_array=array(), $createCHash=FALSE)      {

                        // Setting up internal configuration from config array:
                $this->conf = array();

                        // Information about page for which the indexing takes place
                $this->conf['id'] = $id;                                // Page id      (integer)
                $this->conf['type'] = $type;                    // Page type (integer)
                $this->conf['sys_language_uid'] = $sys_language_uid;    // sys_language UID of the language of the indexing (integer)
                $this->conf['MP'] = $MP;                                // MP variable, if any (Mount Points) (string)
                $this->conf['gr_list'] = '0,-1';        // Group list (hardcoded for now...)

                        // cHash values:
                $this->conf['cHash'] = $createCHash ? $this->makeCHash($cHash_array) : '';      // cHash string for additional parameters
                $this->conf['cHash_array'] = $cHash_array;              // Array of the additional parameters

                        // Set to defaults
                $this->conf['freeIndexUid'] = 0;
                $this->conf['page_cache_reg1'] = '';

                        // Root line uids
                $this->conf['rootline_uids'] = $uidRL;

                        // Configuration of behavior:
                $this->conf['index_externals'] = 1;     // Whether to index external documents like PDF, DOC etc. (if possible)
                $this->conf['index_descrLgd'] = 200;            // Length of description text (max 250, default 200)

                        // Init and start indexing:
                $this->init();
        }

        function backend_setFreeIndexUid($freeIndexUid) {
                $this->conf['freeIndexUid'] = $freeIndexUid;
        }

        function backend_indexAsTYPO3Page($title, $keywords, $description, $content, $charset, $mtime, $crdate=0, $recordUid=0) {

                        // Content of page:
                $this->conf['mtime'] = $mtime;                  // Most recent modification time (seconds) of the content
                $this->conf['crdate'] = $crdate;                // The creation date of the TYPO3 content
                $this->conf['recordUid'] = $recordUid;  // UID of the record, if applicable

                        // Construct fake HTML for parsing:
                $this->conf['content'] = '
                <html>
                        <head>
                                <title>'.htmlspecialchars($title).'</title>
                                <meta name="keywords" content="'.htmlspecialchars($keywords).'" />
                                <meta name="description" content="'.htmlspecialchars($description).'" />
                        </head>
                        <body>
                                '.htmlspecialchars($content).'
                        </body>
                </html>';                                       // Content string (HTML of TYPO3 page)

                        // Initializing charset:
                $this->conf['metaCharset'] = $charset;                  // Character set of content (will be converted to utf-8 during indexing)
                $this->conf['indexedDocTitle'] = '';    // Alternative title for indexing

                        // Index content as if it was a TYPO3 page:
                $this->indexTypo3PageContent();
        }













        /********************************
         *
         * Initialization
         *
         *******************************/

        function init() {
                global $TYPO3_CONF_VARS;

                        // Initializing:
                $this->cHashParams = $this->conf['cHash_array'];
                if (is_array($this->cHashParams) && count($this->cHashParams))  {
                        if ($this->conf['cHash'])       $this->cHashParams['cHash'] = $this->conf['cHash'];     // Add this so that URL's come out right...
                        unset($this->cHashParams['encryptionKey']);             // encryptionKey is added inside TSFE in order to calculate the cHash value and it should NOT be a part of this array!!! If it is it will be exposed in links!!!
                }

                        // Setting phash / phash_grouping which identifies the indexed page based on some of these variables:
                $this->setT3Hashes();

                        // Indexer configuration from Extension Manager interface:
                $this->indexerConfig = unserialize($GLOBALS['TYPO3_CONF_VARS']['EXT']['extConf']['indexed_search']);
                $this->tstamp_minAge = t3lib_div::intInRange($this->indexerConfig['minAge']*3600,0);
                $this->tstamp_maxAge = t3lib_div::intInRange($this->indexerConfig['maxAge']*3600,0);
                $this->maxExternalFiles = t3lib_div::intInRange($this->indexerConfig['maxExternalFiles'],0,1000,5);
                $this->flagBitMask = t3lib_div::intInRange($this->indexerConfig['flagBitMask'],0,255);

                        // Initialize external document parsers:
                        // Example configuration, see ext_localconf.php of this file!
                if ($this->conf['index_externals'])     {
                        $this->initializeExternalParsers();
                }

                        // Initialize lexer (class that deconstructs the text into words):
                        // Example configuration (localconf.php) for this hook: $TYPO3_CONF_VARS['EXTCONF']['indexed_search']['lexer'] = 'EXT:indexed_search/class.lexer.php:&tx_indexedsearch_lexer';
                $lexerObjRef = $TYPO3_CONF_VARS['EXTCONF']['indexed_search']['lexer'] ?
                                                $TYPO3_CONF_VARS['EXTCONF']['indexed_search']['lexer'] :
                                                'EXT:indexed_search/class.lexer.php:&tx_indexedsearch_lexer';
                $this->lexerObj = &t3lib_div::getUserObj($lexerObjRef);
                $this->lexerObj->debug = $this->indexerConfig['debugMode'];

                        // Initialize metaphone hook:
                        // Example configuration (localconf.php) for this hook: $TYPO3_CONF_VARS['EXTCONF']['indexed_search']['metaphone'] = 'EXT:indexed_search/class.doublemetaphone.php:&user_DoubleMetaPhone';
                if ($TYPO3_CONF_VARS['EXTCONF']['indexed_search']['metaphone']) {
                        $this->metaphoneObj = &t3lib_div::getUserObj($TYPO3_CONF_VARS['EXTCONF']['indexed_search']['metaphone']);
                        $this->metaphoneObj->pObj = &$this;
                }

                        // Init charset class:
                $this->csObj = &t3lib_div::makeInstance('t3lib_cs');
        }

        function initializeExternalParsers()    {
                global $TYPO3_CONF_VARS;

                if (is_array($TYPO3_CONF_VARS['EXTCONF']['indexed_search']['external_parsers']))        {
                        foreach($TYPO3_CONF_VARS['EXTCONF']['indexed_search']['external_parsers'] as $extension => $_objRef)    {
                                $this->external_parsers[$extension] = &t3lib_div::getUserObj($_objRef);
                                $this->external_parsers[$extension]->pObj = &$this;

                                        // Init parser and if it returns false, unset its entry again:
                                if (!$this->external_parsers[$extension]->initParser($extension))       {
                                        unset($this->external_parsers[$extension]);
                                }
                        }
                }
        }















        /********************************
         *
         * Indexing; TYPO3 pages (HTML content)
         *
         *******************************/

        function indexTypo3PageContent()        {

                $check = $this->checkMtimeTstamp($this->conf['mtime'], $this->hash['phash']);
                $is_grlist = $this->is_grlist_set($this->hash['phash']);

                if ($check > 0 || !$is_grlist || $this->forceIndexing)  {

                                // Setting message:
                        if ($this->forceIndexing)       {
                                $this->log_setTSlogMessage('Indexing needed, reason: Forced',1);
                        } elseif ($check > 0)   {
                                $this->log_setTSlogMessage('Indexing needed, reason: '.$this->reasons[$check],1);
                        } else {
                                $this->log_setTSlogMessage('Indexing needed, reason: Updates gr_list!',1);
                        }

                                        // Divide into title,keywords,description and body:
                        $this->log_push('Split content','');
                                $this->contentParts = $this->splitHTMLContent($this->conf['content']);
                                if ($this->conf['indexedDocTitle'])     {
                                        $this->contentParts['title'] = $this->conf['indexedDocTitle'];
                                }
                        $this->log_pull();

                                // Calculating a hash over what is to be the actual page content. Maybe this hash should not include title,description and keywords? The bodytext is the primary concern. (on the other hand a changed page-title would make no difference then, so dont!)
                        $this->content_md5h = $this->md5inthash(implode($this->contentParts,''));

                                // This function checks if there is already a page (with gr_list = 0,-1) indexed and if that page has the very same contentHash.
                                // If the contentHash is the same, then we can rest assured that this page is already indexed and regardless of mtime and origContent we don't need to do anything more.
                                // This will also prevent pages from being indexed if a fe_users has logged in and it turns out that the page content is not changed anyway. fe_users logged in should always search with hash_gr_list = "0,-1" OR "[their_group_list]". This situation will be prevented only if the page has been indexed with no user login on before hand. Else the page will be indexed by users until that event. However that does not present a serious problem.
                        $checkCHash = $this->checkContentHash();
                        if (!is_array($checkCHash) || $check===1)       {
                                $Pstart=t3lib_div::milliseconds();

                                $this->log_push('Converting charset of content ('.$this->conf['metaCharset'].') to utf-8','');
                                        $this->charsetEntity2utf8($this->contentParts,$this->conf['metaCharset']);
                                $this->log_pull();

                                                // Splitting words
                                $this->log_push('Extract words from content','');
                                        $splitInWords = $this->procesWordsInArrays($this->contentParts);
                                $this->log_pull();

                                                // Analyse the indexed words.
                                $this->log_push('Analyse the extracted words','');
                                        $indexArr = $this->indexAnalyze($splitInWords);
                                $this->log_pull();

                                                // Submitting page (phash) record
                                $this->log_push('Submitting page','');
                                        $this->submitPage();
                                $this->log_pull();

                                                // Check words and submit to word list if not there
                                $this->log_push('Check word list and submit words','');
                                        $this->checkWordList($indexArr);
                                        $this->submitWords($indexArr,$this->hash['phash']);
                                $this->log_pull();

                                                // Set parsetime
                                $this->updateParsetime($this->hash['phash'],t3lib_div::milliseconds()-$Pstart);

                                                // Checking external files if configured for.
                                $this->log_push('Checking external files','');
                                if ($this->conf['index_externals'])     {
                                        $this->extractLinks($this->conf['content']);
                                }
                                $this->log_pull();
                        } else {
                                $this->updateTstamp($this->hash['phash'],$this->conf['mtime']); // Update the timestatmp
                                $this->update_grlist($checkCHash['phash'],$this->hash['phash']);        // $checkCHash['phash'] is the phash of the result row that is similar to the current phash regarding the content hash.
                                $this->updateRootline();
                                $this->log_setTSlogMessage('Indexing not needed, the contentHash, '.$this->content_md5h.', has not changed. Timestamp, grlist and rootline updated if necessary.');
                        }
                } else {
                        $this->log_setTSlogMessage('Indexing not needed, reason: '.$this->reasons[$check]);
                }
        }

        function splitHTMLContent($content) {

                        // divide head from body ( u-ouh :) )
                $contentArr = $this->defaultContentArray;
                $contentArr['body'] = stristr($content,'<body');
                $headPart = substr($content,0,-strlen($contentArr['body']));

                        // get title
                $this->embracingTags($headPart,'TITLE',$contentArr['title'],$dummy2,$dummy);
                $titleParts = explode(':',$contentArr['title'],2);
                $contentArr['title'] = trim(isset($titleParts[1]) ? $titleParts[1] : $titleParts[0]);

                        // get keywords and description metatags
                for($i=0;$this->embracingTags($headPart,'meta',$dummy,$headPart,$meta[$i]);$i++) { /*nothing*/ }
                for($i=0;isset($meta[$i]);$i++) {
                        $meta[$i] = t3lib_div::get_tag_attributes($meta[$i]);
                        if(stristr($meta[$i]['name'],'keywords')) $contentArr['keywords'].=','.$meta[$i]['content'];
                        if(stristr($meta[$i]['name'],'description')) $contentArr['description'].=','.$meta[$i]['content'];
                }

                        // Process <!--TYPO3SEARCH_begin--> or <!--TYPO3SEARCH_end--> tags:
                $this->typoSearchTags($contentArr['body']);

                        // Get rid of unwanted sections (ie. scripting and style stuff) in body
                $tagList = explode(',',$this->excludeSections);
                foreach($tagList as $tag)       {
                        while($this->embracingTags($contentArr['body'],$tag,$dummy,$contentArr['body'],$dummy2));
                }

                        // remove tags, but first make sure we don't concatenate words by doing it
                $contentArr['body'] = str_replace('<',' <',$contentArr['body']);
                $contentArr['body'] = trim(strip_tags($contentArr['body']));

                $contentArr['keywords'] = trim($contentArr['keywords']);
                $contentArr['description'] = trim($contentArr['description']);

                        // Return array
                return $contentArr;
        }

        function getHTMLcharset($content)       {
                if (eregi('<meta[[:space:]]+[^>]*http-equiv[[:space:]]*=[[:space:]]*["\']CONTENT-TYPE["\'][^>]*>',$content,$reg))       {
                        if (eregi('charset[[:space:]]*=[[:space:]]*([[:alnum:]-]+)',$reg[0],$reg2))     {
                                return $reg2[1];
                        }
                }
        }

        function convertHTMLToUtf8($content,$charset='')        {

                        // Find charset:
                $charset = $charset ? $charset : $this->getHTMLcharset($content);
                $charset = $this->csObj->parse_charset($charset);

                        // Convert charset:
                if ($charset && $charset!=='utf-8')     {
                        $content = $this->csObj->utf8_encode($content, $charset);
                }
                        // Convert entities, assuming document is now UTF-8:
                $content = $this->csObj->entities_to_utf8($content, TRUE);

                return $content;
        }

        function embracingTags($string,$tagName,&$tagContent,&$stringAfter,&$paramList) {
                $endTag = '</'.$tagName.'>';
                $startTag = '<'.$tagName;

                $isTagInText = stristr($string,$startTag);              // stristr used because we want a case-insensitive search for the tag.
                if(!$isTagInText) return false; // if the tag was not found, return false

                list($paramList,$isTagInText) = explode('>',substr($isTagInText,strlen($startTag)),2);
                $afterTagInText = stristr($isTagInText,$endTag);
                if ($afterTagInText)    {
                        $stringBefore = substr($string, 0, strpos(strtolower($string), strtolower($startTag)));
                        $tagContent = substr($isTagInText,0,strlen($isTagInText)-strlen($afterTagInText));
                        $stringAfter = $stringBefore.substr($afterTagInText,strlen($endTag));
                } else {        // If there was no ending tag, the tagContent is blank and anything after the tag it self is returned.
                        $tagContent='';
                        $stringAfter = $isTagInText;
                }

                return true;
        }

        function typoSearchTags(&$body) {
                $expBody = explode('<!--TYPO3SEARCH_',$body);

                if(count($expBody)>1) {
                        $body = '';

                        foreach($expBody as $val)       {
                                $part = explode('-->',$val,2);
                                if(trim($part[0])=='begin') {
                                        $body.= $part[1];
                                        $prev = '';
                                } elseif(trim($part[0])=='end') {
                                        $body.= $prev;
                                } else {
                                        $prev = $val;
                                }
                        }
                        return true;
                } else {
                        return false;
                }
        }

        function extractLinks($content) {

                        // Get links:
                $list = $this->extractHyperLinks($content);

                        // Traverse links:
                foreach($list as $linkInfo)     {

                                // Decode entities:
                        $linkSource = t3lib_div::htmlspecialchars_decode($linkInfo['href']);

                                // Parse URL:
                        $qParts = parse_url($linkSource);

                                // Check for jumpurl (TYPO3 specific thing...)
                        if ($qParts['query'] && strstr($qParts['query'],'jumpurl='))    {
                                parse_str($qParts['query'],$getP);
                                $linkSource = $getP['jumpurl'];
                                $qParts = parse_url($linkSource);       // parse again due to new linkSource!
                        }

                        if ($qParts['scheme'])  {
                                if ($this->indexerConfig['indexExternalURLs'])  {
                                                // Index external URL (http or otherwise)
                                        $this->indexExternalUrl($linkSource);
                                }
                        } elseif (!$qParts['query']) {
                                $localFile = t3lib_div::getFileAbsFileName(PATH_site.$linkSource);
                                if ($localFile && @is_file($localFile)) {
                                                // Index local file:
                                        $this->indexRegularDocument($linkSource);
                                }
                        }
                }
        }

        function extractHyperLinks($string)     {
                if (!is_object($this->htmlParser))      {
                        $this->htmlParser = t3lib_div::makeInstance('t3lib_parseHtml');
                }

                $parts = $this->htmlParser->splitTags('a',$string);
                $list = array();
                foreach($parts as $k => $v)     {
                        if ($k%2)       {
                                $params = $this->htmlParser->get_tag_attributes($v,1);
                                $firstTagName = $this->htmlParser->getFirstTagName($v); // The 'name' of the first tag

                                switch(strtolower($firstTagName))       {
                                        case 'a':
                                                $src = $params[0]['href'];
                                                if ($src)       {
                                                        $list[] = array(
                                                                'tag' => $v,
                                                                'href' => $params[0]['href']
                                                        );
                                                }
                                        break;
                                }
                        }
                }

                return $list;
        }











        /******************************************
         *
         * Indexing; external URL
         *
         ******************************************/

        function indexExternalUrl($externalUrl) {

                        // Parse External URL:
                $qParts = parse_url($externalUrl);
                $fI = pathinfo($qParts['path']);
                $ext = strtolower($fI['extension']);

                        // Get headers:
                $urlHeaders = $this->getUrlHeaders($externalUrl);
                if (stristr($urlHeaders['Content-Type'],'text/html'))   {
                        $content = $this->indexExternalUrl_content = t3lib_div::getUrl($externalUrl);
                        if (strlen($content))   {

                                        // Create temporary file:
                                $tmpFile = t3lib_div::tempnam('EXTERNAL_URL').'.html';
                                t3lib_div::writeFile($tmpFile, $content);

                                        // Index that file:
                                $this->indexRegularDocument($externalUrl, FALSE, $tmpFile, 'html');
                                unlink($tmpFile);
                        }
                }
        }

        function getUrlHeaders($url, $timeout = 2)      {
                $url = parse_url($url);

                if(!in_array($url['scheme'],array('','http')))  return FALSE;

                $fp = fsockopen ($url['host'], ($url['port'] > 0 ? $url['port'] : 80), $errno, $errstr, $timeout);
                if (!$fp)       {
                        return FALSE;
                } else {
                        $msg = "GET ".$url['path'].($url['query'] ? '?'.$url['query'] : '')." HTTP/1.0\r\nHost: ".$url['host']."\r\n\r\n";
                        fputs ($fp, $msg);
                        $d = '';
                        while (!feof($fp)) {
                                $line = fgets ($fp,2048);

                                $d.=$line;
                                if (!strlen(trim($line)))       {
                                        break;
                                }
                        }
                        fclose ($fp);

                                // Compile headers:
                        $headers = t3lib_div::trimExplode(chr(10),$d,1);
                        $retVal = array();
                        foreach($headers as $line)      {
                                list($headKey, $headValue) = explode(':', $line, 2);
                                $retVal[$headKey] = $headValue;
                        }
                        return $retVal;
                }
        }













        /******************************************
         *
         * Indexing; external files (PDF, DOC, etc)
         *
         ******************************************/

        function indexRegularDocument($file, $force=FALSE, $contentTmpFile='', $altExtension='')        {

                        // Init
                $fI = pathinfo($file);
                $ext = $altExtension ? $altExtension : strtolower($fI['extension']);

                        // Create abs-path:
                if (!$contentTmpFile)   {
                        if (!t3lib_div::isAbsPath($file))       {       // Relative, prepend PATH_site:
                                $absFile = t3lib_div::getFileAbsFileName(PATH_site.$file);
                        } else {        // Absolute, pass-through:
                                $absFile = $file;
                        }
                        $absFile = t3lib_div::isAllowedAbsPath($absFile) ? $absFile : '';
                } else {
                        $absFile = $contentTmpFile;
                }

                        // Indexing the document:
                if ($absFile &&  @is_file($absFile))    {
                        if ($this->external_parsers[$ext])      {
                                $mtime = filemtime($absFile);
                                $cParts = $this->fileContentParts($ext,$absFile);

                                foreach($cParts as $cPKey)      {
                                        $this->internal_log = array();
                                        $this->log_push('Index: '.str_replace('.','_',basename($file)).($cPKey?'#'.$cPKey:''),'');
                                        $Pstart = t3lib_div::milliseconds();
                                        $subinfo = array('key' => $cPKey);      // Setting page range. This is "0" (zero) when no division is made, otherwise a range like "1-3"
                                        $phash_arr = $this->file_phash_arr = $this->setExtHashes($file,$subinfo);
                                        $check = $this->checkMtimeTstamp($mtime, $phash_arr['phash']);
                                        if ($check > 0 || $force)       {
                                                if ($check > 0) {
                                                        $this->log_setTSlogMessage('Indexing needed, reason: '.$this->reasons[$check],1);
                                                } else {
                                                        $this->log_setTSlogMessage('Indexing forced by flag',1);
                                                }

                                                        // Check external file counter:
                                                if ($this->externalFileCounter < $this->maxExternalFiles || $force)     {

                                                                        // Divide into title,keywords,description and body:
                                                        $this->log_push('Split content','');
                                                                $contentParts = $this->readFileContent($ext,$absFile,$cPKey);
                                                        $this->log_pull();

                                                        if (is_array($contentParts))    {
                                                                        // Calculating a hash over what is to be the actual content. (see indexTypo3PageContent())
                                                                $content_md5h = $this->md5inthash(implode($contentParts,''));

                                                                if ($this->checkExternalDocContentHash($phash_arr['phash_grouping'], $content_md5h) || $force)  {

                                                                                // Increment counter:
                                                                        $this->externalFileCounter++;

                                                                                // Splitting words
                                                                        $this->log_push('Extract words from content','');
                                                                                $splitInWords = $this->procesWordsInArrays($contentParts);
                                                                        $this->log_pull();

                                                                                // Analyse the indexed words.
                                                                        $this->log_push('Analyse the extracted words','');
                                                                                $indexArr = $this->indexAnalyze($splitInWords);
                                                                        $this->log_pull();

                                                                                // Submitting page (phash) record
                                                                        $this->log_push('Submitting page','');
                                                                                $size = filesize($absFile);
                                                                                $ctime = filemtime($absFile);   // Unfortunately I cannot determine WHEN a file is originally made - so I must return the modification time...
                                                                                $this->submitFilePage($phash_arr,$file,$subinfo,$ext,$mtime,$ctime,$size,$content_md5h,$contentParts);
                                                                        $this->log_pull();

                                                                                // Check words and submit to word list if not there
                                                                        $this->log_push('Check word list and submit words','');
                                                                                $this->checkWordList($indexArr);
                                                                                $this->submitWords($indexArr,$phash_arr['phash']);
                                                                        $this->log_pull();

                                                                                // Set parsetime
                                                                        $this->updateParsetime($phash_arr['phash'],t3lib_div::milliseconds()-$Pstart);
                                                                } else {
                                                                        $this->updateTstamp($phash_arr['phash'],$mtime);        // Update the timestamp
                                                                        $this->log_setTSlogMessage('Indexing not needed, the contentHash, '.$content_md5h.', has not changed. Timestamp updated.');
                                                                }
                                                        } else $this->log_setTSlogMessage('Could not index file! Unsupported extension.');
                                                } else $this->log_setTSlogMessage('The limit of '.$this->maxExternalFiles.' has already been exceeded, so no indexing will take place this time.');
                                        } else $this->log_setTSlogMessage('Indexing not needed, reason: '.$this->reasons[$check]);

                                                // Checking and setting sections:
                #                       $this->submitFile_grlist($phash_arr['phash']);  // Setting a gr_list record if there is none already (set for default fe_group)
                                        $this->submitFile_section($phash_arr['phash']);         // Setting a section-record for the file. This is done also if the file is not indexed. Notice that section records are deleted when the page is indexed.
                                        $this->log_pull();
                                }
                        } else $this->log_setTSlogMessage('Indexing not possible; The extension "'.$ext.'" was not supported.');
                } else $this->log_setTSlogMessage('Indexing not possible; File "'.$absFile.'" not found or valid.');
        }

        function readFileContent($ext,$absFile,$cPKey)  {

                        // Consult relevant external document parser:
                if (is_object($this->external_parsers[$ext]))   {
                        $contentArr = $this->external_parsers[$ext]->readFileContent($ext,$absFile,$cPKey);
                }

                return $contentArr;
        }

        function fileContentParts($ext,$absFile)        {
                $cParts = array(0);

                        // Consult relevant external document parser:
                if (is_object($this->external_parsers[$ext]))   {
                        $cParts = $this->external_parsers[$ext]->fileContentParts($ext,$absFile);
                }

                return $cParts;
        }

        function splitRegularContent($content) {
                $contentArr = $this->defaultContentArray;
                $contentArr['body'] = $content;

                return $contentArr;
        }














        /**********************************
         *
         * Analysing content, Extracting words
         *
         **********************************/

        function charsetEntity2utf8(&$contentArr, $charset)     {

                        // Convert charset if necessary
                reset($contentArr);
                while(list($key,)=each($contentArr)) {
                        if (strlen($contentArr[$key]))  {

                                if ($charset!=='utf-8') {
                                        $contentArr[$key] = $this->csObj->utf8_encode($contentArr[$key], $charset);
                                }

                                        // decode all numeric / html-entities in the string to real characters:
                                $contentArr[$key] = $this->csObj->entities_to_utf8($contentArr[$key],TRUE);
                        }
                }
        }

        function procesWordsInArrays($contentArr)       {

                        // split all parts to words
                reset($contentArr);
                while(list($key,)=each($contentArr)) {
                        $contentArr[$key] = $this->lexerObj->split2Words($contentArr[$key]);
                }

                        // For title, keywords, and description we don't want duplicates:
                $contentArr['title'] = array_unique($contentArr['title']);
                $contentArr['keywords'] = array_unique($contentArr['keywords']);
                $contentArr['description'] = array_unique($contentArr['description']);

                        // Return modified array:
                return $contentArr;
        }

        function bodyDescription($contentArr)   {

                        // Setting description
                $maxL = t3lib_div::intInRange($this->conf['index_descrLgd'],0,255,200);
                if ($maxL)      {
                                // Takes the quadruple lenght first, because whitespace and entities may be removed and thus shorten the string more yet.
        #               $bodyDescription = implode(' ',split('[[:space:],]+',substr(trim($contentArr['body']),0,$maxL*4)));
                        $bodyDescription = str_replace(array(' ',"\t","\r","\n"),' ',$contentArr['body']);

                                // Shorten the string:
                        $bodyDescription = $this->csObj->strtrunc('utf-8', $bodyDescription, $maxL);
                }

                return $bodyDescription;
        }

        function indexAnalyze($content) {
                $indexArr = Array();
                $counter = 0;

                $this->analyzeHeaderinfo($indexArr,$content,'title',7);
                $this->analyzeHeaderinfo($indexArr,$content,'keywords',6);
                $this->analyzeHeaderinfo($indexArr,$content,'description',5);
                $this->analyzeBody($indexArr,$content);

                return ($indexArr);
        }

        function analyzeHeaderinfo(&$retArr,$content,$key,$offset) {
                reset($content[$key]);
                while(list(,$val)=each($content[$key]))  {
                        $val = substr($val,0,60);       // Max 60 - because the baseword varchar IS 60. This MUST be the same.
                        $retArr[$val]['cmp'] = $retArr[$val]['cmp']|pow(2,$offset);
                        $retArr[$val]['count'] = $retArr[$val]['count']+1;
                        $retArr[$val]['hash'] = hexdec(substr(md5($val),0,7));
                        $retArr[$val]['metaphone'] = $this->metaphone($val);
                        $this->wordcount++;
                }
        }

        function analyzeBody(&$retArr,$content) {
                foreach($content['body'] as $key => $val)       {
                        $val = substr($val,0,60);       // Max 60 - because the baseword varchar IS 60. This MUST be the same.
                        if(!isset($retArr[$val])) {
                                $retArr[$val]['first'] = $key;
                                $retArr[$val]['hash'] = hexdec(substr(md5($val),0,7));
                                $retArr[$val]['metaphone'] = $this->metaphone($val);
                        }
                        $retArr[$val]['count'] = $retArr[$val]['count']+1;
                        $this->wordcount++;
                }
        }

        function metaphone($word,$retRaw=FALSE) {

                if (is_object($this->metaphoneObj))     {
                        $tmp = $this->metaphoneObj->metaphone($word, $this->conf['sys_language_uid']);
                } else {
                        $tmp = metaphone($word);
                }

                        // Return raw value?
                if ($retRaw)    return $tmp;

                        // Otherwise create hash and return integer
                if($tmp=='') $ret=0; else $ret=hexdec(substr(md5($tmp),0,7));
                return $ret;
        }
















        /********************************
         *
         * SQL; TYPO3 Pages
         *
         *******************************/

        function submitPage()   {

                        // Remove any current data for this phash:
                $this->removeOldIndexedPages($this->hash['phash']);

                        // setting new phash_row
                $fields = array(
                        'phash' => $this->hash['phash'],
                        'phash_grouping' => $this->hash['phash_grouping'],
                        'cHashParams' => serialize($this->cHashParams),
                        'contentHash' => $this->content_md5h,
                        'data_page_id' => $this->conf['id'],
                        'data_page_reg1' => $this->conf['page_cache_reg1'],
                        'data_page_type' => $this->conf['type'],
                        'data_page_mp' => $this->conf['MP'],
                        'gr_list' => $this->conf['gr_list'],
                        'item_type' => 0,       // TYPO3 page
                        'item_title' => $this->contentParts['title'],
                        'item_description' => $this->bodyDescription($this->contentParts),
                        'item_mtime' => $this->conf['mtime'],
                        'item_size' => strlen($this->conf['content']),
                        'tstamp' => time(),
                        'crdate' => time(),
                        'item_crdate' => $this->conf['crdate'], // Creation date of page
                        'sys_language_uid' => $this->conf['sys_language_uid'],  // Sys language uid of the page. Should reflect which language it DOES actually display!
                        'externalUrl' => 0,
                        'recordUid' => intval($this->conf['recordUid']),
                        'freeIndexUid' => intval($this->conf['freeIndexUid']),
                );
                $GLOBALS['TYPO3_DB']->exec_INSERTquery('index_phash', $fields);

                        // PROCESSING index_section
                $this->submit_section($this->hash['phash'],$this->hash['phash']);

                        // PROCESSING index_grlist
                $this->submit_grlist($this->hash['phash'],$this->hash['phash']);

                        // PROCESSING index_fulltext
                $fields = array(
                        'phash' => $this->hash['phash'],
                        'fulltextdata' => implode(' ', $this->contentParts)
                );
                $GLOBALS['TYPO3_DB']->exec_INSERTquery('index_fulltext', $fields);

                        // PROCESSING index_debug
                if ($this->indexerConfig['debugMode'])  {
                        $fields = array(
                                'phash' => $this->hash['phash'],
                                'debuginfo' => serialize(array(
                                                'cHashParams' => $this->cHashParams,
                                                'external_parsers initialized' => array_keys($this->external_parsers),
                                                'conf' => array_merge($this->conf,array('content'=>substr($this->conf['content'],0,1000))),
                                                'contentParts' => array_merge($this->contentParts,array('body' => substr($this->contentParts['body'],0,1000))),
                                                'logs' => $this->internal_log,
                                                'lexer' => $this->lexerObj->debugString,
                                        ))
                        );
                        $GLOBALS['TYPO3_DB']->exec_INSERTquery('index_debug', $fields);
                }
        }

        function submit_grlist($hash,$phash_x)  {

                        // Setting the gr_list record
                $fields = array(
                        'phash' => $hash,
                        'phash_x' => $phash_x,
                        'hash_gr_list' => $this->md5inthash($this->conf['gr_list']),
                        'gr_list' => $this->conf['gr_list']
                );
                $GLOBALS['TYPO3_DB']->exec_INSERTquery('index_grlist', $fields);
        }

        function submit_section($hash,$hash_t3) {
                $fields = array(
                        'phash' => $hash,
                        'phash_t3' => $hash_t3,
                        'page_id' => intval($this->conf['id'])
                );

                $this->getRootLineFields($fields);

                $GLOBALS['TYPO3_DB']->exec_INSERTquery('index_section', $fields);
        }

        function removeOldIndexedPages($phash)  {
                        // Removing old registrations for all tables. Because the pages are TYPO3 pages there can be nothing else than 1-1 relations here.
                $tableArr = explode(',','index_phash,index_section,index_grlist,index_fulltext,index_debug');
                foreach($tableArr as $table)    {
                        $GLOBALS['TYPO3_DB']->exec_DELETEquery($table, 'phash='.intval($phash));
                }
                        // Removing all index_section records with hash_t3 set to this hash (this includes such records set for external media on the page as well!). The re-insert of these records are done in indexRegularDocument($file).
                $GLOBALS['TYPO3_DB']->exec_DELETEquery('index_section', 'phash_t3='.intval($phash));
        }













        /********************************
         *
         * SQL; External media
         *
         *******************************/


        function submitFilePage($hash,$file,$subinfo,$ext,$mtime,$ctime,$size,$content_md5h,$contentParts)      {

                        // Find item Type:
                $storeItemType = $this->external_parsers[$ext]->ext2itemtype_map[$ext];
                $storeItemType = $storeItemType ? $storeItemType : $ext;

                        // Remove any current data for this phash:
                $this->removeOldIndexedFiles($hash['phash']);

                        // Split filename:
                $fileParts = parse_url($file);

                        // setting new
                $fields = array(
                        'phash' => $hash['phash'],
                        'phash_grouping' => $hash['phash_grouping'],
                        'cHashParams' => serialize($subinfo),
                        'contentHash' => $content_md5h,
                        'data_filename' => $file,
                        'item_type' => $storeItemType,
                        'item_title' => trim($contentParts['title']) ? $contentParts['title'] : basename($file),
                        'item_description' => $this->bodyDescription($contentParts),
                        'item_mtime' => $mtime,
                        'item_size' => $size,
                        'item_crdate' => $ctime,
                        'tstamp' => time(),
                        'crdate' => time(),
                        'gr_list' => $this->conf['gr_list'],
                        'externalUrl' => $fileParts['scheme'] ? 1 : 0,
                        'recordUid' => intval($this->conf['recordUid']),
                        'freeIndexUid' => intval($this->conf['freeIndexUid']),
                );
                $GLOBALS['TYPO3_DB']->exec_INSERTquery('index_phash', $fields);

                        // PROCESSING index_fulltext
                $fields = array(
                        'phash' => $hash['phash'],
                        'fulltextdata' => implode(' ', $contentParts)
                );
                $GLOBALS['TYPO3_DB']->exec_INSERTquery('index_fulltext', $fields);

                        // PROCESSING index_debug
                if ($this->indexerConfig['debugMode'])  {
                        $fields = array(
                                'phash' => $hash['phash'],
                                'debuginfo' => serialize(array(
                                                'cHashParams' => $subinfo,
                                                'contentParts' => array_merge($contentParts,array('body' => substr($contentParts['body'],0,1000))),
                                                'logs' => $this->internal_log,
                                                'lexer' => $this->lexerObj->debugString,
                                        ))
                        );
                        $GLOBALS['TYPO3_DB']->exec_INSERTquery('index_debug', $fields);
                }
        }

        function submitFile_grlist($hash)       {
                        // Testing if there is a gr_list record for a non-logged in user and if so, there is no need to place another one.
                $res = $GLOBALS['TYPO3_DB']->exec_SELECTquery('phash', 'index_grlist', 'phash='.intval($hash).' AND (hash_gr_list='.$this->md5inthash($this->defaultGrList).' OR hash_gr_list='.$this->md5inthash($this->conf['gr_list']).')');
                if (!$GLOBALS['TYPO3_DB']->sql_num_rows($res))  {
                        $this->submit_grlist($hash,$hash);
                }
        }

        function submitFile_section($hash)      {
                        // Testing if there is a section
                $res = $GLOBALS['TYPO3_DB']->exec_SELECTquery('phash', 'index_section', 'phash='.intval($hash).' AND page_id='.intval($this->conf['id']));
                if (!$GLOBALS['TYPO3_DB']->sql_num_rows($res))  {
                        $this->submit_section($hash,$this->hash['phash']);
                }
        }

        function removeOldIndexedFiles($phash)  {

                        // Removing old registrations for tables.
                $tableArr = explode(',','index_phash,index_grlist,index_fulltext,index_debug');
                foreach($tableArr as $table)    {
                        $GLOBALS['TYPO3_DB']->exec_DELETEquery($table, 'phash='.intval($phash));
                }
        }














        /********************************
         *
         * SQL Helper functions
         *
         *******************************/

        function checkMtimeTstamp($mtime,$phash)        {

                        // Select indexed page:
                $res = $GLOBALS['TYPO3_DB']->exec_SELECTquery('item_mtime,tstamp', 'index_phash', 'phash='.intval($phash));
                $out = 0;

                        // If there was an indexing of the page...:
                if ($row = $GLOBALS['TYPO3_DB']->sql_fetch_assoc($res)) {
                        if ($this->tstamp_maxAge && ($row['tstamp']+$this->tstamp_maxAge) < time())     {               // If max age is exceeded, index the page
                                $out = 1;               // The configured max-age was exceeded for the document and thus it's indexed.
                        } else {
                                if (!$this->tstamp_minAge || ($row['tstamp']+$this->tstamp_minAge)<time())      {       // if minAge is not set or if minAge is exceeded, consider at mtime
                                        if ($mtime)     {               // It mtime is set, then it's tested. If not, the page must clearly be indexed.
                                                if ($row['item_mtime'] != $mtime)       {       // And if mtime is different from the index_phash mtime, it's about time to re-index.
                                                        $out = 2;               // The minimum age was exceed and mtime was set and the mtime was different, so the page was indexed.
                                                } else {
                                                        $out = -1;              // mtime matched the document, so no changes detected and no content updated
                                                        if ($this->tstamp_maxAge)       {
                                                                $this->log_setTSlogMessage('Mtime matched, timestamp NOT updated because a maxAge is set ('.($row['tstamp'] + $this->tstamp_maxAge - time()).' seconds to expire time).',1);
                                                        } else {
                                                                $this->updateTstamp($phash);    // Update the timestatmp
                                                                $this->log_setTSlogMessage('Mtime matched, timestamp updated.',1);
                                                        }
                                                }
                                        } else {$out = 3;       }       // The minimum age was exceed, but mtime was not set, so the page was indexed.
                                } else {$out = -2;}                     // The minimum age was not exceeded
                        }
                } else {$out = 4;}      // Page has never been indexed (is not represented in the index_phash table).
                return $out;
        }

        function checkContentHash()     {
                        // With this query the page will only be indexed if it's content is different from the same "phash_grouping" -page.
                $res = $GLOBALS['TYPO3_DB']->exec_SELECTquery('phash', 'index_phash A', 'A.phash_grouping='.intval($this->hash['phash_grouping']).' AND A.contentHash='.intval($this->content_md5h));
                if ($row = $GLOBALS['TYPO3_DB']->sql_fetch_assoc($res)) {
                        return $row;
                }
                return 1;
        }

        function checkExternalDocContentHash($hashGr,$content_md5h)     {
                $res = $GLOBALS['TYPO3_DB']->exec_SELECTquery('*', 'index_phash A', 'A.phash_grouping='.intval($hashGr).' AND A.contentHash='.intval($content_md5h));
                if ($row = $GLOBALS['TYPO3_DB']->sql_fetch_assoc($res)) {
                        return 0;
                }
                return 1;
        }

        function is_grlist_set($phash_x)        {
                $res = $GLOBALS['TYPO3_DB']->exec_SELECTquery('phash_x', 'index_grlist', 'phash_x='.intval($phash_x));
                return $GLOBALS['TYPO3_DB']->sql_num_rows($res);
        }

        function update_grlist($phash,$phash_x) {
                $res = $GLOBALS['TYPO3_DB']->exec_SELECTquery('phash', 'index_grlist', 'phash='.intval($phash).' AND hash_gr_list='.$this->md5inthash($this->conf['gr_list']));
                if (!$GLOBALS['TYPO3_DB']->sql_num_rows($res))  {
                        $this->submit_grlist($phash,$phash_x);
                        $this->log_setTSlogMessage("Inserted gr_list '".$this->conf['gr_list']."' for phash '".$phash."'",1);
                }
        }

        function updateTstamp($phash,$mtime=0)  {
                $updateFields = array(
                        'tstamp' => time()
                );
                if ($mtime)     { $updateFields['item_mtime'] = intval($mtime); }

                $GLOBALS['TYPO3_DB']->exec_UPDATEquery('index_phash', 'phash='.intval($phash), $updateFields);
        }

        function updateParsetime($phash,$parsetime)     {
                $updateFields = array(
                        'parsetime' => intval($parsetime)
                );

                $GLOBALS['TYPO3_DB']->exec_UPDATEquery('index_phash', 'phash='.intval($phash), $updateFields);
        }

        function updateRootline()       {

                $updateFields = array();
                $this->getRootLineFields($updateFields);

                $GLOBALS['TYPO3_DB']->exec_UPDATEquery('index_section', 'page_id='.intval($this->conf['id']), $updateFields);
        }

        function getRootLineFields(&$fieldArr)  {

                $fieldArr['rl0'] = intval($this->conf['rootline_uids'][0]);
                $fieldArr['rl1'] = intval($this->conf['rootline_uids'][1]);
                $fieldArr['rl2'] = intval($this->conf['rootline_uids'][2]);

                if (is_array($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['indexed_search']['addRootLineFields']))    {
                        foreach($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['indexed_search']['addRootLineFields'] as $fieldName => $rootLineLevel)  {
                                $fieldArr[$fieldName] = intval($this->conf['rootline_uids'][$rootLineLevel]);
                        }
                }
        }

        function removeLoginpagesWithContentHash()      {
                $res = $GLOBALS['TYPO3_DB']->exec_SELECTquery('*', 'index_phash A,index_grlist B', '
                                        A.phash=B.phash
                                        AND A.phash_grouping='.intval($this->hash['phash_grouping']).'
                                        AND B.hash_gr_list!='.$this->md5inthash($this->defaultGrList).'
                                        AND A.contentHash='.intval($this->content_md5h));
                while($row = $GLOBALS['TYPO3_DB']->sql_fetch_assoc($res))       {
                        $this->log_setTSlogMessage("The currently indexed page was indexed under no user-login and apparently this page has been indexed under login conditions earlier, but with the SAME content. Therefore the old similar page with phash='".$row['phash']."' are now removed.",1);
                        $this->removeOldIndexedPages($row['phash']);
                }
        }












        /********************************
         *
         * SQL; Submitting words
         *
         *******************************/

        function checkWordList($wl) {
                reset($wl);
                $phashArr = array();
                while(list($key,) = each($wl)) {
                        $phashArr[] = $wl[$key]['hash'];
                }
                if (count($phashArr))   {
                        $cwl = implode(',',$phashArr);
                        $res = $GLOBALS['TYPO3_DB']->exec_SELECTquery('baseword', 'index_words', 'wid IN ('.$cwl.')');

                        if($GLOBALS['TYPO3_DB']->sql_num_rows($res)!=count($wl)) {
                                $this->log_setTSlogMessage('Inserting words: '.(count($wl)-$GLOBALS['TYPO3_DB']->sql_num_rows($res)),1);
                                while($row = $GLOBALS['TYPO3_DB']->sql_fetch_assoc($res)) {
                                        unset($wl[$row['baseword']]);
                                }

                                reset($wl);
                                while(list($key,$val)=each($wl)) {
                                        $insertFields = array(
                                                'wid' => $val['hash'],
                                                'baseword' => $key,
                                                'metaphone' => $val['metaphone']
                                        );
                                                // A duplicate-key error will occur here if a word is NOT unset in the unset() line. However as long as the words in $wl are NOT longer as 60 chars (the baseword varchar is 60 characters...) this is not a problem.
                                        $GLOBALS['TYPO3_DB']->exec_INSERTquery('index_words', $insertFields);
                                }
                        }
                }
        }

        function submitWords($wl,$phash) {
                $GLOBALS['TYPO3_DB']->exec_DELETEquery('index_rel', 'phash='.intval($phash));

                foreach($wl as $val)    {
                        $insertFields = array(
                                'phash' => $phash,
                                'wid' => $val['hash'],
                                'count' => $val['count'],
                                'first' => $val['first'],
                                'freq' => $this->freqMap(($val['count']/$this->wordcount)),
                                'flags' => ($val['cmp'] & $this->flagBitMask)
                        );

                        $GLOBALS['TYPO3_DB']->exec_INSERTquery('index_rel', $insertFields);
                }
        }

        function freqMap($freq) {
                $mapFactor = $this->freqMax*100*$this->freqRange;
                if($freq<1) {
                        $newFreq = $freq*$mapFactor;
                        $newFreq = $newFreq>$this->freqRange?$this->freqRange:$newFreq;
                } else {
                        $newFreq = $freq/$mapFactor;
                }
                return $newFreq;

        }











        /********************************
         *
         * Hashing
         *
         *******************************/

        function setT3Hashes()  {

                        //  Set main array:
                $hArray = array(
                        'id' => (integer)$this->conf['id'],
                        'type' => (integer)$this->conf['type'],
                        'sys_lang' => (integer)$this->conf['sys_language_uid'],
                        'MP' => (string)$this->conf['MP'],
                        'cHash' => $this->cHashParams
                );

                        // Set grouping hash (Identifies a "page" combined of id, type, language, mountpoint and cHash parameters):
                $this->hash['phash_grouping'] = $this->md5inthash(serialize($hArray));

                        // Add gr_list and set plain phash (Subdivision where special page composition based on login is taken into account as well. It is expected that such pages are normally similar regardless of the login.)
                $hArray['gr_list'] = (string)$this->conf['gr_list'];
                $this->hash['phash'] = $this->md5inthash(serialize($hArray));
        }

        function setExtHashes($file,$subinfo=array())   {
                        //  Set main array:
                $hash = array();
                $hArray = array(
                        'file' => $file,
                );

                        // Set grouping hash:
                $hash['phash_grouping'] = $this->md5inthash(serialize($hArray));

                        // Add subinfo
                $hArray['subinfo'] = $subinfo;
                $hash['phash'] = $this->md5inthash(serialize($hArray));

                return $hash;
        }

        function md5inthash($str)       {
                return hexdec(substr(md5($str),0,7));
        }

        function makeCHash($paramArray) {
                $addQueryParams = t3lib_div::implodeArrayForUrl('', $paramArray);

                $pA = t3lib_div::cHashParams($addQueryParams);

                return t3lib_div::shortMD5(serialize($pA));
        }












        /*********************************
         *
         * Internal logging functions
         *
         *********************************/

        function log_push($msg,$key)    {
                if (is_object($GLOBALS['TT']))          $GLOBALS['TT']->push($msg,$key);
        }

        function log_pull()     {
                if (is_object($GLOBALS['TT']))          $GLOBALS['TT']->pull();
        }

        function log_setTSlogMessage($msg, $errorNum=0) {
                if (is_object($GLOBALS['TT']))          $GLOBALS['TT']->setTSlogMessage($msg,$errorNum);
                $this->internal_log[] = $msg;
        }








        /**************************
         *
         * tslib_fe hooks:
         *
         **************************/

        function fe_headerNoCache(&$params, $ref)       {

                        // Requirements are that the crawler is loaded, a crawler session is running and re-indexing requested as processing instruction:
                if (t3lib_extMgm::isLoaded('crawler')
                                && $params['pObj']->applicationData['tx_crawler']['running']
                                && in_array('tx_indexedsearch_reindex', $params['pObj']->applicationData['tx_crawler']['parameters']['procInstructions']))      {

                                // Setting simple log entry:
                        $params['pObj']->applicationData['tx_crawler']['log'][] = 'RE_CACHE (indexed), old status: '.$params['disableAcquireCacheData'];

                                // Disables a look-up for cached page data - thus resulting in re-generation of the page even if cached.
                        $params['disableAcquireCacheData'] = TRUE;
                }
        }
}


if (defined('TYPO3_MODE') && $TYPO3_CONF_VARS[TYPO3_MODE]['XCLASS']['ext/indexed_search/class.indexer.php'])    {
        include_once($TYPO3_CONF_VARS[TYPO3_MODE]['XCLASS']['ext/indexed_search/class.indexer.php']);
}
?>

---