diff --git a/AUTHORS.md b/AUTHORS.md index 050b40f8f8..dbf553b796 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -189,3 +189,11 @@ - Sergio Oller - Will Monroe - Elijah Rippeth + +## Others whose work we've taken and included in NLTK, but who didn't directly contribute it: +### Contributors to the Porter Stemmer +- Martin Porter +- Vivake Gupta +- Barry Wilkins +- Hiranmay Ghosh +- Chris Emerson \ No newline at end of file diff --git a/nltk/stem/porter.py b/nltk/stem/porter.py index ac03067940..48e848f6c8 100644 --- a/nltk/stem/porter.py +++ b/nltk/stem/porter.py @@ -1,93 +1,25 @@ -# Copyright (c) 2002 Vivake Gupta (vivakeATomniscia.org). All rights reserved. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License as -# published by the Free Software Foundation; either version 2 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 -# USA -# -# This software is maintained by Vivake (vivakeATomniscia.org) and is available at: -# http://www.omniscia.org/~vivake/python/PorterStemmer.py -# -# Additional modifications were made to incorporate this module into -# NLTK. All such modifications are marked with "--NLTK--". The NLTK -# version of this module is maintained by NLTK developers, -# and is available via http://nltk.org/ -# -# GNU Linking Exception: -# Using this module statically or dynamically with other modules is -# making a combined work based on this module. Thus, the terms and -# conditions of the GNU General Public License cover the whole combination. -# As a special exception, the copyright holders of this module give -# you permission to combine this module with independent modules to -# produce an executable program, regardless of the license terms of these -# independent modules, and to copy and distribute the resulting -# program under terms of your choice, provided that you also meet, -# for each linked independent module, the terms and conditions of -# the license of that module. An independent module is a module which -# is not derived from or based on this module. If you modify this module, -# you may extend this exception to your version of the module, but you -# are not obliged to do so. If you do not wish to do so, delete this -# exception statement from your version. - """ Porter Stemmer -This is the Porter stemming algorithm, ported to Python from the -version coded up in ANSI C by the author. It follows the algorithm +This is the Porter stemming algorithm. It follows the algorithm presented in Porter, M. "An algorithm for suffix stripping." Program 14.3 (1980): 130-137. -only differing from it at the points marked --DEPARTURE-- and --NEW-- -below. +with some optional deviations that can be turned on or off with the +`mode` argument to the constructor. -For a more faithful version of the Porter algorithm, see +Martin Porter, the algorithm's inventor, maintains a web page about the +algorithm at http://www.tartarus.org/~martin/PorterStemmer/ -Later additions: - - June 2000 - - The 'l' of the 'logi' -> 'log' rule is put with the stem, so that - short stems like 'geo' 'theo' etc work like 'archaeo' 'philo' etc. - - This follows a suggestion of Barry Wilkins, research student at - Birmingham. - - - February 2000 - - the cvc test for not dropping final -e now looks after vc at the - beginning of a word, so are, eve, ice, ore, use keep final -e. In this - test c is any consonant, including w, x and y. This extension was - suggested by Chris Emerson. - - -fully -> -ful treated like -fulness -> -ful, and - -tionally -> -tion treated like -tional -> -tion - - both in Step 2. These were suggested by Hiranmay Ghosh, of New Delhi. - - Invariants proceed, succeed, exceed. Also suggested by Hiranmay Ghosh. - -Additional modifications were made to incorperate this module into -nltk. All such modifications are marked with \"--NLTK--\". +which includes another Python implementation and other implementations +in many languages. """ from __future__ import print_function, unicode_literals -## --NLTK-- -## Declare this module's documentation format. __docformat__ = 'plaintext' import re @@ -95,69 +27,90 @@ from nltk.stem.api import StemmerI from nltk.compat import python_2_unicode_compatible +class _CannotReplaceSuffix(Exception): + pass + @python_2_unicode_compatible class PorterStemmer(StemmerI): - - ## --NLTK-- - ## Add a module docstring """ A word stemmer based on the Porter stemming algorithm. - Porter, M. \"An algorithm for suffix stripping.\" + Porter, M. "An algorithm for suffix stripping." Program 14.3 (1980): 130-137. - - A few minor modifications have been made to Porter's basic - algorithm. See the source code of this module for more - information. - - The Porter Stemmer requires that all tokens have string types. + + See http://www.tartarus.org/~martin/PorterStemmer/ for the homepage + of the algorithm. + + Martin Porter has endorsed several modifications to the Porter + algorithm since writing his original paper, and those extensions are + included in the implementations on his website. Additionally, others + have proposed further improvements to the algorithm, including NLTK + contributors. There are thus three modes that can be selected by + passing the appropriate constant to the class constructor's `mode` + attribute: + + PorterStemmer.ORIGINAL_ALGORITHM + - Implementation that is faithful to the original paper. + + PorterStemmer.MARTIN_EXTENSIONS + - Implementation that only uses the modifications to the + algorithm that are included in the implementations on Martin + Porter's website. He has declared Porter frozen, so the + behaviour of those implementations should never change. + + PorterStemmer.NLTK_EXTENSIONS (default) + - Implementation that includes further improvements devised by + NLTK contributors or taken from other modified implementations + found on the web. + + For the best stemming, you should use the default NLTK_EXTENSIONS + version. However, if you need to get the same results as either the + original algorithm or one of Martin Porter's hosted versions for + compability reasons, you can use one of the other modes instead. """ - - # The main part of the stemming algorithm starts here. - # Note that only lower case sequences are stemmed. Forcing to lower case - # should be done before stem(...) is called. - - def __init__(self): - - ## --NEW-- - ## This is a table of irregular forms. It is quite short, but still - ## reflects the errors actually drawn to Martin Porter's attention over - ## a 20 year period! - ## - ## Extend it as necessary. - ## - ## The form of the table is: - ## { - ## "p1" : ["s11","s12","s13", ... ], - ## "p2" : ["s21","s22","s23", ... ], - ## ... - ## "pn" : ["sn1","sn2","sn3", ... ] - ## } - ## - ## String sij is mapped to paradigm form pi, and the main stemming - ## process is then bypassed. - - irregular_forms = { - "sky" : ["sky", "skies"], - "die" : ["dying"], - "lie" : ["lying"], - "tie" : ["tying"], - "news" : ["news"], - "inning" : ["innings", "inning"], - "outing" : ["outings", "outing"], - "canning" : ["cannings", "canning"], - "howe" : ["howe"], - - # --NEW-- - "proceed" : ["proceed"], - "exceed" : ["exceed"], - "succeed" : ["succeed"], # Hiranmay Ghosh + + # Modes the Stemmer can be instantiated in + NLTK_EXTENSIONS = 'NLTK_EXTENSIONS' + MARTIN_EXTENSIONS = 'MARTIN_EXTENSIONS' + ORIGINAL_ALGORITHM = 'ORIGINAL_ALGORITHM' + + def __init__(self, mode=NLTK_EXTENSIONS): + if mode not in ( + self.NLTK_EXTENSIONS, + self.MARTIN_EXTENSIONS, + self.ORIGINAL_ALGORITHM + ): + raise ValueError( + "Mode must be one of PorterStemmer.NLTK_EXTENSIONS, " + "PorterStemmer.MARTIN_EXTENSIONS, or " + "PorterStemmer.ORIGINAL_ALGORITHM" + ) + + self.mode = mode + + if self.mode == self.NLTK_EXTENSIONS: + # This is a table of irregular forms. It is quite short, + # but still reflects the errors actually drawn to Martin + # Porter's attention over a 20 year period! + irregular_forms = { + "sky" : ["sky", "skies"], + "die" : ["dying"], + "lie" : ["lying"], + "tie" : ["tying"], + "news" : ["news"], + "inning" : ["innings", "inning"], + "outing" : ["outings", "outing"], + "canning" : ["cannings", "canning"], + "howe" : ["howe"], + "proceed" : ["proceed"], + "exceed" : ["exceed"], + "succeed" : ["succeed"], } - self.pool = {} - for key in irregular_forms: - for val in irregular_forms[key]: - self.pool[val] = key + self.pool = {} + for key in irregular_forms: + for val in irregular_forms[key]: + self.pool[val] = key self.vowels = frozenset(['a', 'e', 'i', 'o', 'u']) @@ -171,489 +124,528 @@ def _cons(self, word, i): else: return (not self._cons(word, i - 1)) return True - - def _m(self, word, j): - """m() measures the number of consonant sequences between k0 and j. - if c is a consonant sequence and v a vowel sequence, and <..> - indicates arbitrary presence, - - gives 0 - vc gives 1 - vcvc gives 2 - vcvcvc gives 3 - .... + + def _measure(self, stem): + """Returns the 'measure' of stem, per definition in the paper + + From the paper: + + A consonant will be denoted by c, a vowel by v. A list + ccc... of length greater than 0 will be denoted by C, and a + list vvv... of length greater than 0 will be denoted by V. + Any word, or part of a word, therefore has one of the four + forms: + + CVCV ... C + CVCV ... V + VCVC ... C + VCVC ... V + + These may all be represented by the single form + + [C]VCVC ... [V] + + where the square brackets denote arbitrary presence of their + contents. Using (VC){m} to denote VC repeated m times, this + may again be written as + + [C](VC){m}[V]. + + m will be called the \measure\ of any word or word part when + represented in this form. The case m = 0 covers the null + word. Here are some examples: + + m=0 TR, EE, TREE, Y, BY. + m=1 TROUBLE, OATS, TREES, IVY. + m=2 TROUBLES, PRIVATE, OATEN, ORRERY. """ - n = 0 - i = 0 - while True: - if i > j: - return n - if not self._cons(word, i): - break - i = i + 1 - i = i + 1 - - while True: - while True: - if i > j: - return n - if self._cons(word, i): - break - i = i + 1 - i = i + 1 - n = n + 1 - - while True: - if i > j: - return n - if not self._cons(word, i): - break - i = i + 1 - i = i + 1 - - def _vowelinstem(self, stem): - """vowelinstem(stem) is TRUE <=> stem contains a vowel""" + cv_sequence = '' + + # Construct a string of 'c's and 'v's representing whether each + # character in `stem` is a consonsant or a vowel. + # e.g. 'falafel' becomes 'cvcvcvc', + # 'architecture' becomes 'vcccvcvccvcv' + for i in range(len(stem)): + if self._cons(stem, i): + cv_sequence += 'c' + else: + cv_sequence += 'v' + + # Count the number of 'vc' occurences, which is equivalent to + # the number of 'VC' occurrences in Porter's reduced form in the + # docstring above, which is in turn equivalent to `m` + return cv_sequence.count('vc') + + def _has_positive_measure(self, stem): + return self._measure(stem) > 0 + + def _contains_vowel(self, stem): + """_contains_vowel(stem) is TRUE <=> stem contains a vowel""" for i in range(len(stem)): if not self._cons(stem, i): return True return False - def _doublec(self, word): - """doublec(word) is TRUE <=> word ends with a double consonant""" - if len(word) < 2: - return False - if (word[-1] != word[-2]): - return False - return self._cons(word, len(word)-1) - - def _cvc(self, word, i): - """cvc(i) is TRUE <=> - - a) ( --NEW--) i == 1, and word[0] word[1] is vowel consonant, or - - b) word[i - 2], word[i - 1], word[i] has the form consonant - - vowel - consonant and also if the second c is not w, x or y. this - is used when trying to restore an e at the end of a short word. - e.g. - - cav(e), lov(e), hop(e), crim(e), but - snow, box, tray. + def _ends_cvc(self, word): + """Implements condition *o from the paper + + From the paper: + + *o - the stem ends cvc, where the second c is not W, X or Y + (e.g. -WIL, -HOP). """ - if i == 0: return False # i == 0 never happens perhaps - if i == 1: return (not self._cons(word, 0) and self._cons(word, 1)) - if not self._cons(word, i) or self._cons(word, i-1) or not self._cons(word, i-2): return False - - ch = word[i] - if ch == 'w' or ch == 'x' or ch == 'y': - return False - - return True - - def _step1ab(self, word): - """step1ab() gets rid of plurals and -ed or -ing. e.g. - - caresses -> caress - ponies -> poni - sties -> sti - tie -> tie (--NEW--: see below) - caress -> caress - cats -> cat - - feed -> feed - agreed -> agree - disabled -> disable - - matting -> mat - mating -> mate - meeting -> meet - milling -> mill - messing -> mess - - meetings -> meet + return ( + len(word) >= 3 and + self._cons(word, len(word) - 3) and + not self._cons(word, len(word) - 2) and + self._cons(word, len(word) - 1) and + word[-1] not in ('w', 'x', 'y') + ) + + def _replace_suffix(self, word, suffix, replacement): + """Replaces `suffix` of `word` with `replacement""" + assert word.endswith(suffix), "Given word doesn't end with given suffix" + return word[:-len(suffix)] + replacement + + def _replace_suffix_if(self, word, suffix, replacement, condition): + """If `condition`, replace suffix with replacement, else raise + + `condition` should be a lambda that takes the word and stem as + arguments and returns True or False. """ - if word[-1] == 's': - if word.endswith("sses"): - word = word[:-2] - elif word.endswith("ies"): - if len(word) == 4: - word = word[:-1] - # this line extends the original algorithm, so that - # 'flies'->'fli' but 'dies'->'die' etc - else: - word = word[:-2] - elif word[-2] != 's': - word = word[:-1] - - ed_or_ing_trimmed = False - if word.endswith("ied"): - if len(word) == 4: - word = word[:-1] + if not word.endswith(suffix): + raise _CannotReplaceSuffix("word does not end with suffix") + else: + stem = self._replace_suffix(word, suffix, replacement) + if condition is None or condition(stem): + return stem else: - word = word[:-2] - # this line extends the original algorithm, so that - # 'spied'->'spi' but 'died'->'die' etc - - elif word.endswith("eed"): - if self._m(word, len(word)-4) > 0: - word = word[:-1] - - - elif word.endswith("ed") and self._vowelinstem(word[:-2]): - word = word[:-2] - ed_or_ing_trimmed = True - elif word.endswith("ing") and self._vowelinstem(word[:-3]): - word = word[:-3] - ed_or_ing_trimmed = True - - if ed_or_ing_trimmed: - if word.endswith("at") or word.endswith("bl") or word.endswith("iz"): - word += 'e' - elif self._doublec(word): - if word[-1] not in ['l', 's', 'z']: - word = word[:-1] - elif (self._m(word, len(word)-1) == 1 and self._cvc(word, len(word)-1)): - word += 'e' - + raise _CannotReplaceSuffix("condition not met") + + def _apply_first_possible_rule(self, word, rules): + """Applies the first applicable suffix-removal rule to the word + + Takes a word and a list of suffix-removal rules represented as + 3-tuples, with the first element being the suffix to remove, + the second element being the string to replace it with, and the + final element being the condition for the rule to be applicable, + or None if the rule is unconditional. + """ + for rule in rules: + try: + return self._replace_suffix_if(word, *rule) + except _CannotReplaceSuffix: + pass + return word - + + def _step1a(self, word): + """Implements Step 1a from "An algorithm for suffix stripping" + + From the paper: + + SSES -> SS caresses -> caress + IES -> I ponies -> poni + ties -> ti + SS -> SS caress -> caress + S -> cats -> cat + """ + return self._apply_first_possible_rule(word, [ + ('sses', 'ss', None), # SSES -> SS + + # this NLTK-only rule extends the original algorithm, so + # that 'flies'->'fli' but 'dies'->'die' etc + ( + 'ies', + 'ie', + lambda stem: (self.mode == self.NLTK_EXTENSIONS and + len(word) == 4) + ), + + ('ies', 'i', None), # IES -> I + ('ss', 'ss', None), # SS -> SS + ('s', '', None), # S -> + ]) + + def _step1b(self, word): + """Implements Step 1b from "An algorithm for suffix stripping" + + From the paper: + + (m>0) EED -> EE feed -> feed + agreed -> agree + (*v*) ED -> plastered -> plaster + bled -> bled + (*v*) ING -> motoring -> motor + sing -> sing + + If the second or third of the rules in Step 1b is successful, the following + is done: + + AT -> ATE conflat(ed) -> conflate + BL -> BLE troubl(ed) -> trouble + IZ -> IZE siz(ed) -> size + (*d and not (*L or *S or *Z)) + -> single letter + hopp(ing) -> hop + tann(ed) -> tan + fall(ing) -> fall + hiss(ing) -> hiss + fizz(ed) -> fizz + (m=1 and *o) -> E fail(ing) -> fail + fil(ing) -> file + + The rule to map to a single letter causes the removal of one of the double + letter pair. The -E is put back on -AT, -BL and -IZ, so that the suffixes + -ATE, -BLE and -IZE can be recognised later. This E may be removed in step + 4. + """ + # this NLTK-only block extends the original algorithm, so that + # 'spied'->'spi' but 'died'->'die' etc + if self.mode == self.NLTK_EXTENSIONS: + try: + return self._replace_suffix_if( + word, 'ied', 'ie', lambda stem: len(word) == 4 + ) + except _CannotReplaceSuffix: + pass + + try: + # (m>0) EED -> EE + return self._replace_suffix_if( + word, 'eed', 'ee', lambda stem: self._measure(stem) > 0 + ) + except _CannotReplaceSuffix: + pass + + rule_2_or_3_succeeded = False + for rule in [ + ('ed', '', self._contains_vowel), # (*v*) ED -> + ('ing', '', self._contains_vowel), # (*v*) ING -> + ]: + try: + intermediate_stem = self._replace_suffix_if(word, *rule) + rule_2_or_3_succeeded = True + break + except _CannotReplaceSuffix: + pass + + if not rule_2_or_3_succeeded: + return word + + final_letter = intermediate_stem[-1] + return self._apply_first_possible_rule(intermediate_stem, [ + ('at', 'ate', None), # AT -> ATE + ('bl', 'ble', None), # BL -> BLE + ('iz', 'ize', None), # IZ -> IZE + # (*d and not (*L or *S or *Z)) + # -> single letter + ( + final_letter*2, + final_letter, + lambda stem: final_letter not in ('l', 's', 'z') + ), + # (m=1 and *o) -> E + ( + '', + 'e', + lambda stem: (self._measure(stem) == 1 and + self._ends_cvc(stem)) + ), + ]) + def _step1c(self, word): - """step1c() turns terminal y to i when there is another vowel in the stem. - --NEW--: This has been modified from the original Porter algorithm so that y->i - is only done when y is preceded by a consonant, but not if the stem - is only a single consonant, i.e. - - (*c and not c) Y -> I - - So 'happy' -> 'happi', but - 'enjoy' -> 'enjoy' etc - - This is a much better rule. Formerly 'enjoy'->'enjoi' and 'enjoyment'-> - 'enjoy'. Step 1c is perhaps done too soon; but with this modification that - no longer really matters. - - Also, the removal of the vowelinstem(z) condition means that 'spy', 'fly', - 'try' ... stem to 'spi', 'fli', 'tri' and conflate with 'spied', 'tried', - 'flies' ... + """Implements Step 1c from "An algorithm for suffix stripping" + + From the paper: + + Step 1c + + (*v*) Y -> I happy -> happi + sky -> sky """ - if word[-1] == 'y' and len(word) > 2 and self._cons(word, len(word) - 2): - return word[:-1] + 'i' - else: + def nltk_condition(stem): + """ + This has been modified from the original Porter algorithm so + that y->i is only done when y is preceded by a consonant, + but not if the stem is only a single consonant, i.e. + + (*c and not c) Y -> I + + So 'happy' -> 'happi', but + 'enjoy' -> 'enjoy' etc + + This is a much better rule. Formerly 'enjoy'->'enjoi' and + 'enjoyment'->'enjoy'. Step 1c is perhaps done too soon; but + with this modification that no longer really matters. + + Also, the removal of the contains_vowel(z) condition means + that 'spy', 'fly', 'try' ... stem to 'spi', 'fli', 'tri' and + conflate with 'spied', 'tried', 'flies' ... + """ + return len(stem) > 1 and self._cons(stem, len(stem) - 1) + + def original_condition(stem): + return self._contains_vowel(stem) + + try: + return self._replace_suffix_if( + word, + 'y', + 'i', + nltk_condition if self.mode == self.NLTK_EXTENSIONS + else original_condition + ) + except _CannotReplaceSuffix: return word def _step2(self, word): - """step2() maps double suffices to single ones. - so -ization ( = -ize plus -ation) maps to -ize etc. note that the - string before the suffix must give m() > 0. + """Implements Step 2 from "An algorithm for suffix stripping" + + From the paper: + + Step 2 + + (m>0) ATIONAL -> ATE relational -> relate + (m>0) TIONAL -> TION conditional -> condition + rational -> rational + (m>0) ENCI -> ENCE valenci -> valence + (m>0) ANCI -> ANCE hesitanci -> hesitance + (m>0) IZER -> IZE digitizer -> digitize + (m>0) ABLI -> ABLE conformabli -> conformable + (m>0) ALLI -> AL radicalli -> radical + (m>0) ENTLI -> ENT differentli -> different + (m>0) ELI -> E vileli - > vile + (m>0) OUSLI -> OUS analogousli -> analogous + (m>0) IZATION -> IZE vietnamization -> vietnamize + (m>0) ATION -> ATE predication -> predicate + (m>0) ATOR -> ATE operator -> operate + (m>0) ALISM -> AL feudalism -> feudal + (m>0) IVENESS -> IVE decisiveness -> decisive + (m>0) FULNESS -> FUL hopefulness -> hopeful + (m>0) OUSNESS -> OUS callousness -> callous + (m>0) ALITI -> AL formaliti -> formal + (m>0) IVITI -> IVE sensitiviti -> sensitive + (m>0) BILITI -> BLE sensibiliti -> sensible """ - if len(word) <= 1: # Only possible at this stage given unusual inputs to stem_word like 'oed' - return word - ch = word[-2] - - if ch == 'a': - if word.endswith("ational"): - return word[:-7] + "ate" if self._m(word, len(word)-8) > 0 else word - elif word.endswith("tional"): - return word[:-2] if self._m(word, len(word)-7) > 0 else word - else: - return word - elif ch == 'c': - if word.endswith("enci"): - return word[:-4] + "ence" if self._m(word, len(word)-5) > 0 else word - elif word.endswith("anci"): - return word[:-4] + "ance" if self._m(word, len(word)-5) > 0 else word - else: - return word - elif ch == 'e': - if word.endswith("izer"): - return word[:-1] if self._m(word, len(word)-5) > 0 else word - else: - return word - elif ch == 'l': - if word.endswith("bli"): - return word[:-3] + "ble" if self._m(word, len(word)-4) > 0 else word # --DEPARTURE-- - # To match the published algorithm, replace "bli" with "abli" and "ble" with "able" - elif word.endswith("alli"): - # --NEW-- - if self._m(word, len(word)-5) > 0: - word = word[:-2] - return self._step2(word) - else: - return word - elif word.endswith("fulli"): - return word[:-2] if self._m(word, len(word)-6) else word # --NEW-- - elif word.endswith("entli"): - return word[:-2] if self._m(word, len(word)-6) else word - elif word.endswith("eli"): - return word[:-2] if self._m(word, len(word)-4) else word - elif word.endswith("ousli"): - return word[:-2] if self._m(word, len(word)-6) else word - else: - return word - elif ch == 'o': - if word.endswith("ization"): - return word[:-7] + "ize" if self._m(word, len(word)-8) else word - elif word.endswith("ation"): - return word[:-5] + "ate" if self._m(word, len(word)-6) else word - elif word.endswith("ator"): - return word[:-4] + "ate" if self._m(word, len(word)-5) else word - else: - return word - elif ch == 's': - if word.endswith("alism"): - return word[:-3] if self._m(word, len(word)-6) else word - elif word.endswith("ness"): - if word.endswith("iveness"): - return word[:-4] if self._m(word, len(word)-8) else word - elif word.endswith("fulness"): - return word[:-4] if self._m(word, len(word)-8) else word - elif word.endswith("ousness"): - return word[:-4] if self._m(word, len(word)-8) else word - else: - return word - else: - return word - elif ch == 't': - if word.endswith("aliti"): - return word[:-3] if self._m(word, len(word)-6) else word - elif word.endswith("iviti"): - return word[:-5] + "ive" if self._m(word, len(word)-6) else word - elif word.endswith("biliti"): - return word[:-6] + "ble" if self._m(word, len(word)-7) else word - else: - return word - elif ch == 'g': # --DEPARTURE-- - if word.endswith("logi"): - return word[:-1] if self._m(word, len(word) - 4) else word # --NEW-- (Barry Wilkins) - # To match the published algorithm, pass len(word)-5 to _m instead of len(word)-4 - else: - return word - - else: - return word + if self.mode == self.NLTK_EXTENSIONS: + # Instead of applying the ALLI -> AL rule after '(a)bli' per + # the published algorithm, instead we apply it first, and, + # if it succeeds, run the result through step2 again. + try: + stem = self._replace_suffix_if( + word, + 'alli', + 'al', + self._has_positive_measure + ) + return self._step2(stem) + except _CannotReplaceSuffix: + pass + + bli_rule = ('bli', 'ble', self._has_positive_measure) + abli_rule = ('abli', 'able', self._has_positive_measure) + + rules = [ + ('ational', 'ate', self._has_positive_measure), + ('tional', 'tion', self._has_positive_measure), + ('enci', 'ence', self._has_positive_measure), + ('anci', 'ance', self._has_positive_measure), + ('izer', 'ize', self._has_positive_measure), + + abli_rule if self.mode == self.ORIGINAL_ALGORITHM else bli_rule, + + ('alli', 'al', self._has_positive_measure), + ('entli', 'ent', self._has_positive_measure), + ('eli', 'e', self._has_positive_measure), + ('ousli', 'ous', self._has_positive_measure), + ('ization', 'ize', self._has_positive_measure), + ('ation', 'ate', self._has_positive_measure), + ('ator', 'ate', self._has_positive_measure), + ('alism', 'al', self._has_positive_measure), + ('iveness', 'ive', self._has_positive_measure), + ('fulness', 'ful', self._has_positive_measure), + ('ousness', 'ous', self._has_positive_measure), + ('aliti', 'al', self._has_positive_measure), + ('iviti', 'ive', self._has_positive_measure), + ('biliti', 'ble', self._has_positive_measure), + ] + + if self.mode == self.NLTK_EXTENSIONS: + rules.append( + ('fulli', 'ful', self._has_positive_measure) + ) + + # The 'l' of the 'logi' -> 'log' rule is put with the stem, + # so that short stems like 'geo' 'theo' etc work like + # 'archaeo' 'philo' etc. + rules.append(( + "logi", + "log", + lambda stem: self._has_positive_measure(word[:-3]) + )) + + if self.mode == self.MARTIN_EXTENSIONS: + rules.append( + ("logi", "log", self._has_positive_measure) + ) + + return self._apply_first_possible_rule(word, rules) def _step3(self, word): - """step3() deals with -ic-, -full, -ness etc. similar strategy to step2.""" - - ch = word[-1] - - if ch == 'e': - if word.endswith("icate"): - return word[:-3] if self._m(word, len(word)-6) else word - elif word.endswith("ative"): - return word[:-5] if self._m(word, len(word)-6) else word - elif word.endswith("alize"): - return word[:-3] if self._m(word, len(word)-6) else word - else: - return word - elif ch == 'i': - if word.endswith("iciti"): - return word[:-3] if self._m(word, len(word)-6) else word - else: - return word - elif ch == 'l': - if word.endswith("ical"): - return word[:-2] if self._m(word, len(word)-5) else word - elif word.endswith("ful"): - return word[:-3] if self._m(word, len(word)-4) else word - else: - return word - elif ch == 's': - if word.endswith("ness"): - return word[:-4] if self._m(word, len(word)-5) else word - else: - return word - - else: - return word + """Implements Step 3 from "An algorithm for suffix stripping" + + From the paper: + + Step 3 + + (m>0) ICATE -> IC triplicate -> triplic + (m>0) ATIVE -> formative -> form + (m>0) ALIZE -> AL formalize -> formal + (m>0) ICITI -> IC electriciti -> electric + (m>0) ICAL -> IC electrical -> electric + (m>0) FUL -> hopeful -> hope + (m>0) NESS -> goodness -> good + """ + return self._apply_first_possible_rule(word, [ + ('icate', 'ic', self._has_positive_measure), + ('ative', '', self._has_positive_measure), + ('alize', 'al', self._has_positive_measure), + ('iciti', 'ic', self._has_positive_measure), + ('ical', 'ic', self._has_positive_measure), + ('ful', '', self._has_positive_measure), + ('ness', '', self._has_positive_measure), + ]) def _step4(self, word): - """step4() takes off -ant, -ence etc., in context vcvc.""" - - if len(word) <= 1: # Only possible at this stage given unusual inputs to stem_word like 'oed' - return word - - ch = word[-2] - - if ch == 'a': - if word.endswith("al"): - return word[:-2] if self._m(word, len(word)-3) > 1 else word - else: - return word - elif ch == 'c': - if word.endswith("ance"): - return word[:-4] if self._m(word, len(word)-5) > 1 else word - elif word.endswith("ence"): - return word[:-4] if self._m(word, len(word)-5) > 1 else word - else: - return word - elif ch == 'e': - if word.endswith("er"): - return word[:-2] if self._m(word, len(word)-3) > 1 else word - else: - return word - elif ch == 'i': - if word.endswith("ic"): - return word[:-2] if self._m(word, len(word)-3) > 1 else word - else: - return word - elif ch == 'l': - if word.endswith("able"): - return word[:-4] if self._m(word, len(word)-5) > 1 else word - elif word.endswith("ible"): - return word[:-4] if self._m(word, len(word)-5) > 1 else word - else: - return word - elif ch == 'n': - if word.endswith("ant"): - return word[:-3] if self._m(word, len(word)-4) > 1 else word - elif word.endswith("ement"): - return word[:-5] if self._m(word, len(word)-6) > 1 else word - elif word.endswith("ment"): - return word[:-4] if self._m(word, len(word)-5) > 1 else word - elif word.endswith("ent"): - return word[:-3] if self._m(word, len(word)-4) > 1 else word - else: - return word - elif ch == 'o': - if word.endswith("sion") or word.endswith("tion"): # slightly different logic to all the other cases - return word[:-3] if self._m(word, len(word)-4) > 1 else word - elif word.endswith("ou"): - return word[:-2] if self._m(word, len(word)-3) > 1 else word - else: - return word - elif ch == 's': - if word.endswith("ism"): - return word[:-3] if self._m(word, len(word)-4) > 1 else word - else: - return word - elif ch == 't': - if word.endswith("ate"): - return word[:-3] if self._m(word, len(word)-4) > 1 else word - elif word.endswith("iti"): - return word[:-3] if self._m(word, len(word)-4) > 1 else word - else: - return word - elif ch == 'u': - if word.endswith("ous"): - return word[:-3] if self._m(word, len(word)-4) > 1 else word - else: - return word - elif ch == 'v': - if word.endswith("ive"): - return word[:-3] if self._m(word, len(word)-4) > 1 else word - else: - return word - elif ch == 'z': - if word.endswith("ize"): - return word[:-3] if self._m(word, len(word)-4) > 1 else word - else: - return word - else: - return word - - def _step5(self, word): - """step5() removes a final -e if m() > 1, and changes -ll to -l if - m() > 1. + """Implements Step 4 from "An algorithm for suffix stripping" + + Step 4 + + (m>1) AL -> revival -> reviv + (m>1) ANCE -> allowance -> allow + (m>1) ENCE -> inference -> infer + (m>1) ER -> airliner -> airlin + (m>1) IC -> gyroscopic -> gyroscop + (m>1) ABLE -> adjustable -> adjust + (m>1) IBLE -> defensible -> defens + (m>1) ANT -> irritant -> irrit + (m>1) EMENT -> replacement -> replac + (m>1) MENT -> adjustment -> adjust + (m>1) ENT -> dependent -> depend + (m>1 and (*S or *T)) ION -> adoption -> adopt + (m>1) OU -> homologou -> homolog + (m>1) ISM -> communism -> commun + (m>1) ATE -> activate -> activ + (m>1) ITI -> angulariti -> angular + (m>1) OUS -> homologous -> homolog + (m>1) IVE -> effective -> effect + (m>1) IZE -> bowdlerize -> bowdler + + The suffixes are now removed. All that remains is a little + tidying up. """ - if word[-1] == 'e': - a = self._m(word, len(word)-1) - if a > 1 or (a == 1 and not self._cvc(word, len(word)-2)): - word = word[:-1] - if word.endswith('ll') and self._m(word, len(word)-1) > 1: - word = word[:-1] - - return word - - def stem_word(self, p, i=0, j=None): + measure_gt_1 = lambda stem: self._measure(stem) > 1 + + return self._apply_first_possible_rule(word, [ + ('al', '', measure_gt_1), + ('ance', '', measure_gt_1), + ('ence', '', measure_gt_1), + ('er', '', measure_gt_1), + ('ic', '', measure_gt_1), + ('able', '', measure_gt_1), + ('ible', '', measure_gt_1), + ('ant', '', measure_gt_1), + ('ement', '', measure_gt_1), + ('ment', '', measure_gt_1), + ('ent', '', measure_gt_1), + + # (m>1 and (*S or *T)) ION -> + ( + 'ion', + '', + lambda stem: self._measure(stem) > 1 and stem[-1] in ('s', 't') + ), + + ('ou', '', measure_gt_1), + ('ism', '', measure_gt_1), + ('ate', '', measure_gt_1), + ('iti', '', measure_gt_1), + ('ous', '', measure_gt_1), + ('ive', '', measure_gt_1), + ('ize', '', measure_gt_1), + ]) + + def _step5a(self, word): + """Implements Step 5a from "An algorithm for suffix stripping" + + From the paper: + + Step 5a + + (m>1) E -> probate -> probat + rate -> rate + (m=1 and not *o) E -> cease -> ceas """ - Returns the stem of p, or, if i and j are given, the stem of p[i:j+1]. + return self._apply_first_possible_rule(word, [ + ('e', '', lambda stem: self._measure(stem) > 1), + ( + 'e', + '', + lambda stem: ( + self._measure(stem) == 1 and + not self._ends_cvc(stem) + ) + ) + ]) + + def _step5b(self, word): + """Implements Step 5a from "An algorithm for suffix stripping" + + From the paper: + + Step 5b + + (m > 1 and *d and *L) -> single letter + controll -> control + roll -> roll """ - ## --NLTK-- - if j is None and i == 0: - word = p - else: - if j is None: - j = len(p) - 1 - word = p[i:j+1] + # The rule is expressed in an overcomplicated way in Porter's + # paper, but all it means it that double-l should become + # single-l. It could've been written more straightforwardly as: + # + # (m > 1) LL -> L + return self._apply_first_possible_rule(word, [ + ('ll', 'l', lambda stem: self._measure(stem) > 1) + ]) - if word in self.pool: + def stem(self, word): + stem = word.lower() + + if self.mode == self.NLTK_EXTENSIONS and word in self.pool: return self.pool[word] - if len(word) <= 2: - return word # --DEPARTURE-- - # With this line, strings of length 1 or 2 don't go through the - # stemming process, although no mention is made of this in the - # published algorithm. Remove the line to match the published - # algorithm. - - word = self._step1ab(word) - word = self._step1c(word) - word = self._step2(word) - word = self._step3(word) - word = self._step4(word) - word = self._step5(word) - return word + if self.mode != self.ORIGINAL_ALGORITHM and len(word) <= 2: + # With this line, strings of length 1 or 2 don't go through + # the stemming process, although no mention is made of this + # in the published algorithm. + return word - def _adjust_case(self, word, stem): - lower = word.lower() + stem = self._step1a(stem) + stem = self._step1b(stem) + stem = self._step1c(stem) + stem = self._step2(stem) + stem = self._step3(stem) + stem = self._step4(stem) + stem = self._step5a(stem) + stem = self._step5b(stem) + + return stem - ret = "" - for x in range(len(stem)): - if lower[x] == stem[x]: - ret += word[x] - else: - ret += stem[x] - - return ret - - ## --NLTK-- - ## Don't use this procedure; we want to work with individual - ## tokens, instead. (commented out the following procedure) - #def stem(self, text): - # parts = re.split("(\W+)", text) - # numWords = (len(parts) + 1)/2 - # - # ret = "" - # for i in xrange(numWords): - # word = parts[2 * i] - # separator = "" - # if ((2 * i) + 1) < len(parts): - # separator = parts[(2 * i) + 1] - # - # stem = self.stem_word(string.lower(word), 0, len(word) - 1) - # ret = ret + self.adjust_case(word, stem) - # ret = ret + separator - # return ret - - ## --NLTK-- - ## Define a stem() method that implements the StemmerI interface. - def stem(self, word): - stem = self.stem_word(word.lower(), 0, len(word) - 1) - return self._adjust_case(word, stem) - - ## --NLTK-- - ## Add a string representation function def __repr__(self): return '' -## --NLTK-- -## This test procedure isn't applicable. -#if __name__ == '__main__': -# p = PorterStemmer() -# if len(sys.argv) > 1: -# for f in sys.argv[1:]: -# with open(f, 'r') as infile: -# while 1: -# w = infile.readline() -# if w == '': -# break -# w = w[:-1] -# print(p.stem(w)) - -##--NLTK-- -## Added a demo() function - def demo(): """ A demonstration of the porter stemmer on a sample from @@ -686,7 +678,3 @@ def demo(): print('-Results-'.center(70).replace(' ', '*').replace('-', ' ')) print(results) print('*'*70) - -##--NLTK-- - -