Documentation TYPO3 par Ameos |
00001 <?php 00002 /*************************************************************** 00003 * Copyright notice 00004 * 00005 * (c) 2001-2006 Kasper Skaarhoj (kasperYYYY@typo3.com) 00006 * All rights reserved 00007 * 00008 * This script is part of the TYPO3 project. The TYPO3 project is 00009 * free software; you can redistribute it and/or modify 00010 * it under the terms of the GNU General Public License as published by 00011 * the Free Software Foundation; either version 2 of the License, or 00012 * (at your option) any later version. 00013 * 00014 * The GNU General Public License can be found at 00015 * http://www.gnu.org/copyleft/gpl.html. 00016 * A copy is found in the textfile GPL.txt and important notices to the license 00017 * from the author is found in LICENSE.txt distributed with these scripts. 00018 * 00019 * 00020 * This script is distributed in the hope that it will be useful, 00021 * but WITHOUT ANY WARRANTY; without even the implied warranty of 00022 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 00023 * GNU General Public License for more details. 00024 * 00025 * This copyright notice MUST APPEAR in all copies of the script! 00026 ***************************************************************/ 00075 class tx_indexed_search_extparse { 00076 00077 // This value is also overridden from config. 00078 var $pdf_mode = -20; // zero: whole PDF file is indexed in one. positive value: Indicates number of pages at a time, eg. "5" would means 1-5,6-10,.... Negative integer would indicate (abs value) number of groups. Eg "3" groups of 10 pages would be 1-4,5-8,9-10 00079 00080 // This array is configured in initialization: 00081 var $app = array(); 00082 var $ext2itemtype_map = array(); 00083 var $supportedExtensions = array(); 00084 00085 var $pObj; // Reference to parent object (indexer class) 00086 00087 00094 function initParser($extension) { 00095 00096 // Then read indexer-config and set if appropriate: 00097 $indexerConfig = unserialize($GLOBALS['TYPO3_CONF_VARS']['EXT']['extConf']['indexed_search']); 00098 00099 // If windows, apply extension to tool name: 00100 $exe = (TYPO3_OS == 'WIN') ? '.exe' : ''; // lg 00101 $extOK = FALSE; 00102 $mainExtension = ''; 00103 00104 // Ignore extensions 00105 $ignoreExtensions = t3lib_div::trimExplode(',', strtolower($indexerConfig['ignoreExtensions']),1); 00106 if (in_array($extension, $ignoreExtensions)) { 00107 $this->pObj->log_setTSlogMessage('Extension "'.$extension.'" was set to be ignored.',1); 00108 return FALSE; 00109 } 00110 00111 // Switch on file extension: 00112 switch($extension) { 00113 case 'pdf': 00114 // PDF 00115 if ($indexerConfig['pdftools']) { 00116 $pdfPath = ereg_replace("\/$",'',$indexerConfig['pdftools']).'/'; 00117 if (ini_get('safe_mode') || (@is_file($pdfPath.'pdftotext'.$exe) && @is_file($pdfPath.'pdfinfo'.$exe))) { 00118 $this->app['pdfinfo'] = $pdfPath.'pdfinfo'.$exe; 00119 $this->app['pdftotext'] = $pdfPath.'pdftotext'.$exe; 00120 // PDF mode: 00121 $this->pdf_mode = t3lib_div::intInRange($indexerConfig['pdf_mode'],-100,100); 00122 $extOK = TRUE; 00123 } else $this->pObj->log_setTSlogMessage("PDF tools was not found in paths '".$pdfPath."pdftotext' and/or '".$pdfPath."pdfinfo'",3); 00124 } else $this->pObj->log_setTSlogMessage('PDF tools disabled',1); 00125 break; 00126 case 'doc': 00127 // Catdoc 00128 if ($indexerConfig['catdoc']) { 00129 $catdocPath = ereg_replace("\/$",'',$indexerConfig['catdoc']).'/'; 00130 if (ini_get('safe_mode') || @is_file($catdocPath.'catdoc'.$exe)) { 00131 $this->app['catdoc'] = $catdocPath.'catdoc'.$exe; 00132 $extOK = TRUE; 00133 } else $this->pObj->log_setTSlogMessage("'catdoc' tool for reading Word-files was not found in path '".$catdocPath."catdoc'",3); 00134 } else $this->pObj->log_setTSlogMessage('catdoc tools (Word-files) disabled',1); 00135 break; 00136 case 'pps': // MS PowerPoint(?) 00137 case 'ppt': // MS PowerPoint 00138 // ppthtml 00139 if ($indexerConfig['ppthtml']) { 00140 $ppthtmlPath = ereg_replace('\/$','',$indexerConfig['ppthtml']).'/'; 00141 if (ini_get('safe_mode') || @is_file($ppthtmlPath.'ppthtml'.$exe)){ 00142 $this->app['ppthtml'] = $ppthtmlPath.'ppthtml'.$exe; 00143 $extOK = TRUE; 00144 } else $this->pObj->log_setTSlogMessage("'ppthtml' tool for reading Powerpoint-files was not found in path '".$ppthtmlPath."ppthtml'",3); 00145 } else $this->pObj->log_setTSlogMessage('ppthtml tools (Powerpoint-files) disabled',1); 00146 break; 00147 case 'xls': // MS Excel 00148 // Xlhtml 00149 if ($indexerConfig['xlhtml']) { 00150 $xlhtmlPath = ereg_replace('\/$','',$indexerConfig['xlhtml']).'/'; 00151 if (ini_get('safe_mode') || @is_file($xlhtmlPath.'xlhtml'.$exe)){ 00152 $this->app['xlhtml'] = $xlhtmlPath.'xlhtml'.$exe; 00153 $extOK = TRUE; 00154 } else $this->pObj->log_setTSlogMessage("'xlhtml' tool for reading Excel-files was not found in path '".$xlhtmlPath."xlhtml'",3); 00155 } else $this->pObj->log_setTSlogMessage('xlhtml tools (Excel-files) disabled',1); 00156 break; 00157 case 'sxc': // Open Office Calc. 00158 case 'sxi': // Open Office Impress 00159 case 'sxw': // Open Office Writer 00160 case 'ods': // Oasis OpenDocument Spreadsheet 00161 case 'odp': // Oasis OpenDocument Presentation 00162 case 'odt': // Oasis OpenDocument Text 00163 if ($indexerConfig['unzip']) { 00164 $unzipPath = preg_replace('/\/$/','',$indexerConfig['unzip']).'/'; 00165 if (ini_get('safe_mode') || @is_file($unzipPath.'unzip'.$exe)) { 00166 $this->app['unzip'] = $unzipPath.'unzip'.$exe; 00167 $extOK = TRUE; 00168 } else $this->pObj->log_setTSlogMessage("'unzip' tool for reading OpenOffice.org-files was not found in path '".$unzipPath."unzip'",3); 00169 } else $this->pObj->log_setTSlogMessage('unzip tool (OpenOffice.org-files) disabled',1); 00170 break; 00171 case 'rtf': 00172 // Catdoc 00173 if ($indexerConfig['unrtf']) { 00174 $unrtfPath = ereg_replace("\/$",'',$indexerConfig['unrtf']).'/'; 00175 if (ini_get('safe_mode') || @is_file($unrtfPath.'unrtf'.$exe)) { 00176 $this->app['unrtf'] = $unrtfPath.'unrtf'.$exe; 00177 $extOK = TRUE; 00178 } else $this->pObj->log_setTSlogMessage("'unrtf' tool for reading RTF-files was not found in path '".$unrtfPath."unrtf'",3); 00179 } else $this->pObj->log_setTSlogMessage('unrtf tool (RTF-files) disabled',1); 00180 break; 00181 case 'txt': // Raw text 00182 case 'csv': // Raw text 00183 case 'xml': // PHP strip-tags() 00184 case 'tif': // PHP EXIF 00185 $extOK = TRUE; 00186 break; 00187 case 'html': // PHP strip-tags() 00188 case 'htm': // PHP strip-tags() 00189 $extOK = TRUE; 00190 $mainExtension = 'html'; // making "html" the common "item_type" 00191 break; 00192 case 'jpg': // PHP EXIF 00193 case 'jpeg': // PHP EXIF 00194 $extOK = TRUE; 00195 $mainExtension = 'jpeg'; // making "jpeg" the common item_type 00196 break; 00197 } 00198 00199 // If extension was OK: 00200 if ($extOK) { 00201 $this->supportedExtensions[$extension] = TRUE; 00202 $this->ext2itemtype_map[$extension] = $mainExtension ? $mainExtension : $extension; 00203 return TRUE; 00204 } 00205 } 00206 00214 function softInit($extension) { 00215 switch($extension) { 00216 case 'pdf': // PDF 00217 case 'doc': // MS Word files 00218 case 'pps': // MS PowerPoint 00219 case 'ppt': // MS PowerPoint 00220 case 'xls': // MS Excel 00221 case 'sxc': // Open Office Calc. 00222 case 'sxi': // Open Office Impress 00223 case 'sxw': // Open Office Writer 00224 case 'ods': // Oasis OpenDocument Spreadsheet 00225 case 'odp': // Oasis OpenDocument Presentation 00226 case 'odt': // Oasis OpenDocument Text 00227 case 'rtf': // RTF documents 00228 case 'txt': // ASCII Text documents 00229 case 'html': // HTML 00230 case 'htm': // HTML 00231 case 'csv': // Comma Separated Values 00232 case 'xml': // Generic XML 00233 case 'jpg': // Jpeg images (EXIF comment) 00234 case 'jpeg': // Jpeg images (EXIF comment) 00235 case 'tif': // TIF images (EXIF comment) 00236 return TRUE; 00237 break; 00238 } 00239 } 00240 00247 function searchTypeMediaTitle($extension) { 00248 00249 // Read indexer-config 00250 $indexerConfig = unserialize($GLOBALS['TYPO3_CONF_VARS']['EXT']['extConf']['indexed_search']); 00251 00252 // Ignore extensions 00253 $ignoreExtensions = t3lib_div::trimExplode(',', strtolower($indexerConfig['ignoreExtensions']),1); 00254 if (in_array($extension, $ignoreExtensions)) { 00255 return FALSE; 00256 } 00257 00258 // Switch on file extension: 00259 switch($extension) { 00260 case 'pdf': 00261 // PDF 00262 if ($indexerConfig['pdftools']) { 00263 return 'PDF'; 00264 } 00265 break; 00266 case 'doc': 00267 // Catdoc 00268 if ($indexerConfig['catdoc']) { 00269 return 'MS Word'; 00270 } 00271 break; 00272 case 'pps': // MS PowerPoint(?) 00273 case 'ppt': // MS PowerPoint 00274 // ppthtml 00275 if ($indexerConfig['ppthtml']) { 00276 return 'MS Powerpoint'; 00277 } 00278 break; 00279 case 'xls': // MS Excel 00280 // Xlhtml 00281 if ($indexerConfig['xlhtml']) { 00282 return 'MS Excel'; 00283 } 00284 break; 00285 case 'sxc': // Open Office Calc. 00286 case 'sxi': // Open Office Impress 00287 case 'sxw': // Open Office Writer 00288 case 'ods': // Oasis OpenDocument Spreadsheet 00289 case 'odp': // Oasis OpenDocument Presentation 00290 case 'odt': // Oasis OpenDocument Text 00291 if ($indexerConfig['unzip']) { 00292 return 'Open Office'; 00293 } 00294 break; 00295 case 'rtf': 00296 // Catdoc 00297 if ($indexerConfig['unrtf']) { 00298 return 'RTF'; 00299 } 00300 break; 00301 case 'html': // PHP strip-tags() 00302 case 'jpeg': // PHP EXIF 00303 case 'txt': // Raw text 00304 case 'csv': // Raw text 00305 case 'xml': // PHP strip-tags() 00306 case 'tif': // PHP EXIF 00307 return strtoupper($extension); 00308 break; 00309 // NO entry (duplicates or blank): 00310 case 'htm': // PHP strip-tags() 00311 case 'jpg': // PHP EXIF 00312 default: 00313 break; 00314 } 00315 } 00316 00323 function isMultiplePageExtension($extension) { 00324 // Switch on file extension: 00325 switch((string)$extension) { 00326 case 'pdf': 00327 return TRUE; 00328 break; 00329 } 00330 } 00331 00332 00333 00334 00335 00336 00337 00338 00339 00340 /************************ 00341 * 00342 * Reading documents (for parsing) 00343 * 00344 ************************/ 00345 00354 function readFileContent($ext,$absFile,$cPKey) { 00355 unset($contentArr); 00356 00357 // Return immediately if initialization didn't set support up: 00358 if (!$this->supportedExtensions[$ext]) return FALSE; 00359 00360 // Switch by file extension 00361 switch ($ext) { 00362 case 'pdf': 00363 if ($this->app['pdfinfo']) { 00364 // Getting pdf-info: 00365 $cmd = $this->app['pdfinfo'].' "'.$absFile.'"'; 00366 exec($cmd,$res); 00367 $pdfInfo = $this->splitPdfInfo($res); 00368 unset($res); 00369 if (intval($pdfInfo['pages'])) { 00370 list($low,$high) = explode('-',$cPKey); 00371 00372 // Get pdf content: 00373 $tempFileName = t3lib_div::tempnam('Typo3_indexer'); // Create temporary name 00374 @unlink ($tempFileName); // Delete if exists, just to be safe. 00375 $cmd = $this->app['pdftotext'].' -f '.$low.' -l '.$high.' -enc UTF-8 -q "'.$absFile.'" '.$tempFileName; 00376 exec($cmd); 00377 if (@is_file($tempFileName)) { 00378 $content = t3lib_div::getUrl($tempFileName); 00379 unlink($tempFileName); 00380 } else { 00381 $this->pObj->log_setTSlogMessage('PDFtoText Failed on this document: '.$absFile.". Maybe the PDF file is locked for printing or encrypted.",2); 00382 } 00383 if (strlen($content)) { 00384 $contentArr = $this->pObj->splitRegularContent($this->removeEndJunk($content)); 00385 } 00386 } 00387 } 00388 break; 00389 case 'doc': 00390 if ($this->app['catdoc']) { 00391 $cmd = $this->app['catdoc'].' -d utf-8 "'.$absFile.'"'; 00392 exec($cmd,$res); 00393 $content = implode(chr(10),$res); 00394 unset($res); 00395 $contentArr = $this->pObj->splitRegularContent($this->removeEndJunk($content)); 00396 } 00397 break; 00398 case 'pps': 00399 case 'ppt': 00400 if ($this->app['ppthtml']) { 00401 $cmd = $this->app['ppthtml'].' "'.$absFile.'"'; 00402 exec($cmd,$res); 00403 $content = implode(chr(10),$res); 00404 unset($res); 00405 $content = $this->pObj->convertHTMLToUtf8($content); 00406 $contentArr = $this->pObj->splitHTMLContent($this->removeEndJunk($content)); 00407 $contentArr['title'] = basename($absFile); // Make sure the title doesn't expose the absolute path! 00408 } 00409 break; 00410 case 'xls': 00411 if ($this->app['xlhtml']) { 00412 $cmd = $this->app['xlhtml'].' -nc -te "'.$absFile.'"'; 00413 exec($cmd,$res); 00414 $content = implode(chr(10),$res); 00415 unset($res); 00416 $content = $this->pObj->convertHTMLToUtf8($content); 00417 $contentArr = $this->pObj->splitHTMLContent($this->removeEndJunk($content)); 00418 $contentArr['title'] = basename($absFile); // Make sure the title doesn't expose the absolute path! 00419 } 00420 break; 00421 case 'sxi': 00422 case 'sxc': 00423 case 'sxw': 00424 case 'ods': 00425 case 'odp': 00426 case 'odt': 00427 if ($this->app['unzip']) { 00428 // Read content.xml: 00429 $cmd = $this->app['unzip'].' -p "'.$absFile.'" content.xml'; 00430 exec($cmd,$res); 00431 $content_xml = implode(chr(10),$res); 00432 unset($res); 00433 00434 // Read meta.xml: 00435 $cmd = $this->app['unzip'].' -p "'.$absFile.'" meta.xml'; 00436 exec($cmd, $res); 00437 $meta_xml = implode(chr(10),$res); 00438 unset($res); 00439 00440 $utf8_content = trim(strip_tags(str_replace('<',' <',$content_xml))); 00441 $contentArr = $this->pObj->splitRegularContent($utf8_content); 00442 $contentArr['title'] = basename($absFile); // Make sure the title doesn't expose the absolute path! 00443 00444 // Meta information 00445 $metaContent = t3lib_div::xml2tree($meta_xml); 00446 $metaContent = $metaContent['office:document-meta'][0]['ch']['office:meta'][0]['ch']; 00447 if (is_array($metaContent)) { 00448 $contentArr['title'] = $metaContent['dc:title'][0]['values'][0] ? $metaContent['dc:title'][0]['values'][0] : $contentArr['title']; 00449 $contentArr['description'] = $metaContent['dc:subject'][0]['values'][0].' '.$metaContent['dc:description'][0]['values'][0]; 00450 00451 // Keywords collected: 00452 if (is_array($metaContent['meta:keywords'][0]['ch']['meta:keyword'])) { 00453 foreach ($metaContent['meta:keywords'][0]['ch']['meta:keyword'] as $kwDat) { 00454 $contentArr['keywords'].= $kwDat['values'][0].' '; 00455 } 00456 } 00457 } 00458 } 00459 break; 00460 case 'rtf': 00461 if ($this->app['unrtf']) { 00462 $cmd = $this->app['unrtf'].' "'.$absFile.'"'; 00463 exec($cmd,$res); 00464 $fileContent = implode(chr(10),$res); 00465 unset($res); 00466 $fileContent = $this->pObj->convertHTMLToUtf8($fileContent); 00467 $contentArr = $this->pObj->splitHTMLContent($fileContent); 00468 } 00469 break; 00470 case 'txt': 00471 case 'csv': // Raw text 00472 $content = t3lib_div::getUrl($absFile); 00473 // TODO: Auto-registration of charset???? -> utf-8 (Current assuming western europe...) 00474 $content = $this->pObj->convertHTMLToUtf8($content, 'iso-8859-1'); 00475 $contentArr = $this->pObj->splitRegularContent($content); 00476 $contentArr['title'] = basename($absFile); // Make sure the title doesn't expose the absolute path! 00477 break; 00478 case 'html': 00479 case 'htm': 00480 $fileContent = t3lib_div::getUrl($absFile); 00481 $fileContent = $this->pObj->convertHTMLToUtf8($fileContent); 00482 $contentArr = $this->pObj->splitHTMLContent($fileContent); 00483 break; 00484 case 'xml': // PHP strip-tags() 00485 $fileContent = t3lib_div::getUrl($absFile); 00486 00487 // Finding charset: 00488 eregi('^[[:space:]]*<\?xml[^>]+encoding[[:space:]]*=[[:space:]]*["\'][[:space:]]*([[:alnum:]_-]+)[[:space:]]*["\']',substr($fileContent,0,200),$reg); 00489 $charset = $reg[1] ? $this->pObj->csObj->parse_charset($reg[1]) : 'utf-8'; 00490 00491 // Converting content: 00492 $fileContent = $this->pObj->convertHTMLToUtf8(strip_tags(str_replace('<',' <',$fileContent)), $charset); 00493 $contentArr = $this->pObj->splitRegularContent($fileContent); 00494 $contentArr['title'] = basename($absFile); // Make sure the title doesn't expose the absolute path! 00495 break; 00496 case 'jpg': // PHP EXIF 00497 case 'jpeg': // PHP EXIF 00498 case 'tif': // PHP EXIF 00499 $exif = exif_read_data($absFile, 'IFD0'); 00500 if ($exif) { 00501 $comment = trim($exif['COMMENT'][0].' '.$exif['ImageDescription']); // The comments in JPEG files are utf-8, while in Tif files they are 7-bit ascii. 00502 } else { 00503 $comment = ''; 00504 } 00505 $contentArr = $this->pObj->splitRegularContent($comment); 00506 $contentArr['title'] = basename($absFile); // Make sure the title doesn't expose the absolute path! 00507 break; 00508 default: 00509 return false; 00510 break; 00511 } 00512 // If no title (and why should there be...) then the file-name is set as title. This will raise the hits considerably if the search matches the document name. 00513 if (is_array($contentArr) && !$contentArr['title']) { 00514 $contentArr['title'] = str_replace('_',' ',basename($absFile)); // Substituting "_" for " " because many filenames may have this instead of a space char. 00515 } 00516 00517 return $contentArr; 00518 } 00519 00528 function fileContentParts($ext,$absFile) { 00529 $cParts = array(0); 00530 switch ($ext) { 00531 case 'pdf': 00532 // Getting pdf-info: 00533 $cmd = $this->app['pdfinfo'].' "'.$absFile.'"'; 00534 exec($cmd,$res); 00535 $pdfInfo = $this->splitPdfInfo($res); 00536 unset($res); 00537 00538 if (intval($pdfInfo['pages'])) { 00539 $cParts = array(); 00540 00541 // Calculate mode 00542 if ($this->pdf_mode>0) { 00543 $iter = ceil($pdfInfo['pages']/$this->pdf_mode); 00544 } else { 00545 $iter = t3lib_div::intInRange(abs($this->pdf_mode),1,$pdfInfo['pages']); 00546 } 00547 00548 // Traverse and create intervals. 00549 for ($a=0;$a<$iter;$a++) { 00550 $low = floor($a*($pdfInfo['pages']/$iter))+1; 00551 $high = floor(($a+1)*($pdfInfo['pages']/$iter)); 00552 $cParts[] = $low.'-'.$high; 00553 } 00554 } 00555 break; 00556 } 00557 return $cParts; 00558 } 00559 00568 function splitPdfInfo($pdfInfoArray) { 00569 $res = array(); 00570 if (is_array($pdfInfoArray)) { 00571 foreach($pdfInfoArray as $line) { 00572 $parts = explode(':',$line,2); 00573 if (count($parts)>1 && trim($parts[0])) { 00574 $res[strtolower(trim($parts[0]))] = trim($parts[1]); 00575 } 00576 } 00577 } 00578 return $res; 00579 } 00580 00587 function removeEndJunk($string) { 00588 return trim(ereg_replace('['.chr(10).chr(12).']*$','',$string)); 00589 } 00590 00591 00592 00593 00594 00595 00596 00597 00598 00599 00600 00601 00602 /************************ 00603 * 00604 * Backend analyzer 00605 * 00606 ************************/ 00607 00614 function getIcon($extension) { 00615 if ($extension=='htm') $extension = 'html'; 00616 if ($extension=='jpeg') $extension = 'jpg'; 00617 return 'EXT:indexed_search/pi/res/'.$extension.'.gif'; 00618 } 00619 } 00620 00621 if (defined('TYPO3_MODE') && $TYPO3_CONF_VARS[TYPO3_MODE]['XCLASS']['ext/indexed_search/class.external_parser.php']) { 00622 include_once($TYPO3_CONF_VARS[TYPO3_MODE]['XCLASS']['ext/indexed_search/class.external_parser.php']); 00623 } 00624 ?>