Keyword Relevancy Algorithms

A small and simple algorithm in order to find relevant keywords in a text works as follows:

1. initialising array of stop words and keywords,
2. text preprocessing, such as lowercasing and removing peculiars,
3. determining general text statistics, such as number of characters,
4. removing stop words,
5. exploding text to keyword array,
6. iterating keyword array in order to count earch word,
7. sorting and delivering result.

This is a reference implementation of this algorithm in JavaScript:

///////////////////////////////////////////////////////////////////////////// // // The following code snippet works on the text from document.f.text.value // Within this JavaScript solution, there is no sorting! // ///////////////////////////////////////////////////////////////////////////// // init var aStopWords = new Array ("about", "all", "alone", "also", "am", "and", "as", "at", "because", "before", "beside", "besides", "between", "but", "by", "etc", "for", "i", "of", "on", "other", "others", "so", "than", "that", "though", "to", "too", "trough", "until"); var aWords = new Array (); var aKeywords = new Array (); // the text var sText = document.f.text.value; // total character count var iCharCount = sText.length; // remove line breaks sText = sText.replace(/\s/g, ' '); // convert to lowercase sText = sText.toLowerCase(); // remove peculiars sText = sText.replace(/[^a-zA-Z0-9äöüß]/g, ' '); // total word count aWords = sText.split(" "); iWordCount = aWords.length; var iCharCountWithout = 0; // count words for (var x = 0; x < aWords.length; x++) { iCharCountWithout += aWords[x].length; } aWords = new Array(); // remove stop words for (var m = 0; m < aStopWords.length; m++) { sText = sText.replace(' ' + aStopWords[m] + ' ', ' '); } // explode to array of words aWords = sText.split(" "); // every word for (var x = 0; x < aWords.length; x++) { // trim the word var s = aWords[x].replace (/^\s+/, '').replace (/\s+$/, ''); // if already in array if (aKeywords[s] != undefined) { // then increase count of this word aKeywords[s]++; } // if not counted yet else { if (s != '') { aKeywords[s] = 1; } } } // result sAlert = "Found keywords:"; n = 1; for (var sKey in aKeywords) { iNumber = aKeywords[sKey]; fQuotient = Math.round(100 * (iNumber / iWordCount), 2); sAlert = sAlert + "\n" + iNumber; sAlert = sAlert + " times (" + fQuotient + " %): "; sAlert = sAlert + sKey; n++; // break the loop if more then 10 results if (n > 10) break; } // alerting result alert(sAlert); // end

Download this JavaScript, embedded in HTML as ZIP file: algorithm_js.zip

An implementation of the algorithm in PHP:

///////////////////////////////////////////////////////////////////////////// // // The following code snippet works on a text delivered via $_POST["text"] // ///////////////////////////////////////////////////////////////////////////// // init $aStopWords = array ('about', 'all', 'alone', 'also', 'am', 'and', 'as', 'at', 'because', 'before', 'beside', 'besides', 'between', 'but', 'by', 'etc', 'for', 'i', 'of', 'on', 'other', 'others', 'so', 'than', 'that', 'though', 'to', 'too', 'trough', 'until'); $aWords = array (); $aKeywords = array (); // total character count $iCharCount = strlen($_POST["text"]); // remove line breaks $sText = ereg_replace("[\r\t\n]", ' ', $_POST["text"]); // decode UTF-8 $sText = utf8_decode($sText); // convert to lowercase $sText = strtolower($sText); // remove peculiars $sText = preg_replace('/[^a-z0-9äöüß&;#]/', ' ', $sText); // total word count $aWords = explode(" ", $sText); $iWordCount = count($aWords); $iCharCountWithout = 0; // count words for ($x = 0; $x < count($aWords); $x++) { $iCharCountWithout += strlen($aWords[$x]); } unset ($aWords); // remove stop words for ($m = 0; $m < count($aStopWords); $m++) { $sText = str_replace(' ' . $aStopWords[$m] . ' ', ' ', $sText); } // reduce spaces $sText = preg_replace('/^\s*$/', ' ', $sText); // explode to array of words $aWords = explode(" ", $sText); // every word for ($x = 0; $x < count($aWords); $x++) { // if already in array if (isset ($aKeywords[$aWords[$x]])) { // then increase count of this word $aKeywords[$aWords[$x]]++; } // if not counted yet else { if (trim($aWords[$x]) != '') { $aKeywords[$aWords[$x]] = 1; } } } // sort arsort($aKeywords); // result echo '<table>'; echo '<tr><th>Count</th>; echo '<th>Percentage</th>; echo '<th>Found keyword</th></tr>'; $x = 0; while ($iNumber = current($aKeywords)) { $iNumber = intval($iNumber); $sKey = key($aKeywords); $fQuotient = number_format(round(100 * ($iNumber / $iWordCount), 2), 2); echo '<tr><td>' . $iNumber . ' </td>'; echo '<td>' . $fQuotient . ' %</td>'; echo '<td>' . $sKey . '</td></tr>'; $x++; next($aKeywords); } echo '</table>'; // end

Download this PHP code as ZIP file: algorithm_php.zip