<?php
	/*
	** File: create_index
	** Description: 
	** Version: $Revision: 1.10 $
	** Created: 9/2/99
	** Author: Leon Atkinson
	** Email: leon@clearink.com
	** CVS Author: $Author: leon $
	** Last Revised: $Date: 2000/02/02 18:59:48 $
	**
	** Log : $Log: create_index,v $
	** Log : Revision 1.10  2000/02/02 18:59:48  leon
	** Log : Fixing bugs reported by Ted Henigson.
	** Log :
	** Log : Revision 1.9  2000/01/05 21:40:08  leon
	** Log : Added a bit of code to check that ScreenIndex is writeable.
	** Log :
	** Log : Revision 1.8  1999/12/09 03:23:16  jj
	** Log : Changed prepareText function (selectively) to cleanText so that there would be
	** Log : no name conflicts (per bug found by John Ashton).
	** Log :
	** Log : Revision 1.7  1999/12/09 03:13:46  jj
	** Log : Fixed bug found by John Ashton (missing semicolon).
	** Log :
	** Log : Revision 1.6  1999/11/08 22:32:23  leon
	** Log : Added prepareText() function to use everywhere we were using htmlentities.
	** Log :
	** Log : Revision 1.5  1999/11/04 21:35:39  leon
	** Log : Cleaned up the use of paths.  Paths are now all set in index.php3 as
	** Log : constants.  Example settings for Apache/Unix, Apache/Win32, and IIS.
	** Log :
	** Log : Revision 1.4  1999/10/13 00:42:26  chris
	** Log : Moving text into constants in preparation of localization support.
	** Log :
	** Log : Revision 1.3  1999/10/12 23:14:50  leon
	** Log : Moved localization constants into 'english' module.  Added constant to
	** Log : control whether unregistered shoppers may checkout.
	** Log :
	** Log : Revision 1.2  1999/10/12 00:39:09  chris
	** Log : Moving text into constants in preparation of localization support.
	** Log :
	** Log : Revision 1.1  1999/09/09 20:34:19  leon
	** Log : Added support for an index-based search.
	** Log : Constants in global_settings controls whether the index is used or
	** Log : not.  You can build/re-build the index through a link on the
	** Log : admin menu.
	** Log :
	**
	** Copyright (c) 1999 Working Dogs.  All rights reserved.
	*/

	$ScreenAppearanceLimit = count($ScreenInfo) * SEARCH_MAXIMUM_FREQUENCY / 100;

	function cleanText($text)
	{
		//replace multiple spaces with a single space
		$text = ereg_replace("[[:space:]]+", " ", $text);
		
		//add linefeeds before and after HTML tags and PHP tags
		$text = ereg_replace("<", "\n<", $text);
		$text = ereg_replace(">", ">\n", $text);

		//add linefeeds around PHP tags
		$text = ereg_replace("<\?", "\n<?\n", $text);
		$text = ereg_replace("\?>", "\n?>\n", $text);
		
		return($text);
	}


	/*
	** Loop through ScreenInfo
	**
	** Why is the code below repeated instead of functionalized, you ask?
	** Apparently GLOBAL works fine for 1D arrays, but it chokes on 2D
	** arrays.  So, we can't get at $ScreenInfo inside a function.  And
	** it seems like a waste to pass ScreenInfo into a function as an
	** argument.  Let's remember to test this functionality in PHP4.
	*/
	$tempContent = "";
	
	//exclude this screen
	unset($ScreenInfo['create_index']); 

	for(reset($ScreenInfo); $screen = key($ScreenInfo); next($ScreenInfo))
	{
		print(L_CREATEINDEX_CHECKING . "$screen<BR>\n");
		flush();
	
		/*
		** tokenize screen description ($ScreenInfo[$screen][SI_DESCRIPTION1])
		*/
		$Line = cleanText($ScreenInfo[$screen][SI_DESCRIPTION]);

		// use strtok to get each word
		for($token = strtok(trim(strtolower($Line . " <end>")), " .,!?'\"");
			$token != "<end>";
			$token = strtok(" .,!?'\""))
		{
			if(strlen($token) > SEARCH_MINIMUM_WORD_LENGTH)
			{
				//count that this token has appeared once
				//on this screen
				$ScreenIndex[$token][$screen]++;
			}
		}

		/*
		** tokenize screen description ($ScreenInfo[$screen][SI_KEYWORDS])
		*/
		$Line = cleanText($ScreenInfo[$screen][SI_KEYWORDS]);

		// use strtok to get each word
		for($token = strtok(trim(strtolower($Line . " <end>")), " .,!?'\"");
			$token != "<end>";
			$token = strtok(" .,!?'\""))
		{
			if(strlen($token) > SEARCH_MINIMUM_WORD_LENGTH)
			{
				//count that this token has appeared once
				//on this screen
				$ScreenIndex[$token][$screen]++;
			}
		}

	
		/*
		** tokenize screen body ($screen)
		*/
		$tempContent = file(APPLICATION_ROOT . "/modules/screens/$screen");
		$tempContent = implode(" ", $tempContent);		
		$Line = split("\n", cleanText($tempContent));
		$LineCount = count($Line);
		$inPHP = FALSE;
		for($n=0; $n < $LineCount; $n++)
		{
			if($Line[$n] == '<?')
			{
				$inPHP = TRUE;
			}
			elseif($Line[$n] == '?>')
			{
				$inPHP = FALSE;
			}
			elseif(($inPHP == FALSE) AND (!eregi("^<.*>$", $Line[$n])))
			{
				/*
				** use strtok to get each word
				*/
				for($token = strtok(trim(strtolower($Line[$n] . " <end>")), " .,!?'\"");
					$token != "<end>";
					$token = strtok(" .,!?'\""))
				{
					if(strlen($token) > SEARCH_MINIMUM_WORD_LENGTH)
					{
						//count that this token has appeared once
						//on this screen
						$ScreenIndex[$token][$screen]++;
					}
				}
			}
		}
	}


	/*
	** Take out high-frequence words
	**
	** The way this works is to loop over each token entry in the array.  The number
	** of screens this token appears on is counted.  If the token appears on too many
	** screens, then it's dropped.  Notice that each time a token appears on a
	** particular screen, it's counted, but that count is not being tested here.  It's
	** used for ranking search results (making the naive assumption that a page
	** with 10 occurences of a token is more interesting than one that mentions it
	** once).
	*/
	print(L_CREATEINDEX_DROPPING);
	print(L_CREATEINDEX_RATIO . ": $ScreenAppearanceLimit/" . count($ScreenInfo) . "<BR>\n");

	for(reset($ScreenIndex); $token=key($ScreenIndex); next($ScreenIndex))
	{
		if(count($ScreenIndex[$token]) > $ScreenAppearanceLimit)
		{
			print("Taking out $token (".count($ScreenIndex[$token]).") <BR>\n");
			unset($ScreenIndex[$token]);
		}
	}


	$outerStarted = FALSE;

	$ArrayDef = "<?php\n\n";
	
	$ArrayDef .= "/*\n";
	$ArrayDef .= "** File: ScreenIndex\n";
	$ArrayDef .= "** Created: ".date("m/d/Y")."\n";
	$ArrayDef .= "**\n";
	$ArrayDef .= "** This file was created by the create_index screen.  Any\n";
	$ArrayDef .= "** edits you make to it will be lost when the index is rebuilt.\n";
	$ArrayDef .= "**\n";
	$ArrayDef .= "*/\n";
	
	$ArrayDef .= "\$ScreenIndex = array(\n";

	for(reset($ScreenIndex); $token=key($ScreenIndex); next($ScreenIndex))
	{
		$innerStarted = FALSE;

		if($outerStarted)
		{
			$ArrayDef .= "\n\t),\n";
		}
	
		$ArrayDef .= "\t'$token'=>array(\n";
		
		for(reset($ScreenIndex[$token]); $screen = key($ScreenIndex[$token]); next($ScreenIndex[$token]))
		{
			if($innerStarted)
			{
				$ArrayDef .= ",\n";
			}

			$ArrayDef .= "\t\t'$screen'=>'" . $ScreenIndex[$token][$screen] . "'";

			$innerStarted = TRUE;
		}

		$outerStarted = TRUE;		
	}

	$ArrayDef .= "\n\t)\n";
	
	$ArrayDef .= "\n);\n";
	$ArrayDef .= "?>\n";
	
	if(DEBUG)
	{
		print("<PRE>");
		print(prepareText($ArrayDef));
		print("</PRE>");
	}
	
	//re-create ScreenIndex module
	if($fp = fopen(APPLICATION_ROOT . "/modules/include/ScreenIndex", "w"))
	{
		fputs($fp, $ArrayDef);
		fclose($fp);
	}
	else
	{
		print("Unable to write new ScreenIndex.  Did you chmod it to allow the ");
		print("Web server to write to it?");
	}
?>
