From e89631a26cfb1ff4383c0248f92a95c7896849dd Mon Sep 17 00:00:00 2001 From: Mark Amery Date: Sat, 9 Jan 2016 15:48:20 +0000 Subject: [PATCH 01/17] [PorterStemmer] Remove obsolete copyright and maintenance notice --- nltk/stem/porter.py | 36 ------------------------------------ 1 file changed, 36 deletions(-) diff --git a/nltk/stem/porter.py b/nltk/stem/porter.py index ac03067940..046b4bcb53 100644 --- a/nltk/stem/porter.py +++ b/nltk/stem/porter.py @@ -1,43 +1,7 @@ -# Copyright (c) 2002 Vivake Gupta (vivakeATomniscia.org). All rights reserved. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License as -# published by the Free Software Foundation; either version 2 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 -# USA -# -# This software is maintained by Vivake (vivakeATomniscia.org) and is available at: -# http://www.omniscia.org/~vivake/python/PorterStemmer.py -# # Additional modifications were made to incorporate this module into # NLTK. All such modifications are marked with "--NLTK--". The NLTK # version of this module is maintained by NLTK developers, # and is available via http://nltk.org/ -# -# GNU Linking Exception: -# Using this module statically or dynamically with other modules is -# making a combined work based on this module. Thus, the terms and -# conditions of the GNU General Public License cover the whole combination. -# As a special exception, the copyright holders of this module give -# you permission to combine this module with independent modules to -# produce an executable program, regardless of the license terms of these -# independent modules, and to copy and distribute the resulting -# program under terms of your choice, provided that you also meet, -# for each linked independent module, the terms and conditions of -# the license of that module. An independent module is a module which -# is not derived from or based on this module. If you modify this module, -# you may extend this exception to your version of the module, but you -# are not obliged to do so. If you do not wish to do so, delete this -# exception statement from your version. """ Porter Stemmer From 3af726f2cbfbabd2e580e4be907c8168ec531400 Mon Sep 17 00:00:00 2001 From: Mark Amery Date: Sat, 9 Jan 2016 15:54:02 +0000 Subject: [PATCH 02/17] [PorterStemmer] Remove trailing whitespace and mysterious trailing comment --- nltk/stem/porter.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/nltk/stem/porter.py b/nltk/stem/porter.py index 046b4bcb53..fc9f42efa0 100644 --- a/nltk/stem/porter.py +++ b/nltk/stem/porter.py @@ -544,13 +544,6 @@ def stem_word(self, p, i=0, j=None): if word in self.pool: return self.pool[word] - if len(word) <= 2: - return word # --DEPARTURE-- - # With this line, strings of length 1 or 2 don't go through the - # stemming process, although no mention is made of this in the - # published algorithm. Remove the line to match the published - # algorithm. - word = self._step1ab(word) word = self._step1c(word) word = self._step2(word) @@ -650,7 +643,3 @@ def demo(): print('-Results-'.center(70).replace(' ', '*').replace('-', ' ')) print(results) print('*'*70) - -##--NLTK-- - - From 9d72aa4baf83367134ea5132d2f52a432ad412dd Mon Sep 17 00:00:00 2001 From: Mark Amery Date: Sat, 9 Jan 2016 15:55:35 +0000 Subject: [PATCH 03/17] [PorterStemmer] Purge gratuitous departure comments in preparation for rewrite --- nltk/stem/porter.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/nltk/stem/porter.py b/nltk/stem/porter.py index fc9f42efa0..9052a04239 100644 --- a/nltk/stem/porter.py +++ b/nltk/stem/porter.py @@ -589,28 +589,9 @@ def stem(self, word): stem = self.stem_word(word.lower(), 0, len(word) - 1) return self._adjust_case(word, stem) - ## --NLTK-- - ## Add a string representation function def __repr__(self): return '' -## --NLTK-- -## This test procedure isn't applicable. -#if __name__ == '__main__': -# p = PorterStemmer() -# if len(sys.argv) > 1: -# for f in sys.argv[1:]: -# with open(f, 'r') as infile: -# while 1: -# w = infile.readline() -# if w == '': -# break -# w = w[:-1] -# print(p.stem(w)) - -##--NLTK-- -## Added a demo() function - def demo(): """ A demonstration of the porter stemmer on a sample from From 20005549f7d546a13565333bcc2df5adb28680c0 Mon Sep 17 00:00:00 2001 From: Mark Amery Date: Sat, 9 Jan 2016 16:05:29 +0000 Subject: [PATCH 04/17] [PorterStemmer] Remove stem_word from PorterStemmer. Breaks backwards compatability! Prior to this change, the public API of the PorterStemmer was a mess. NLTK's version was based off Vivake Gupta's implementation at http://tartarus.org/~martin/PorterStemmer/python.txt, endorsed by Martin himself at http://tartarus.org/~martin/PorterStemmer/. However, Gupta's implementation is a shoddy port of Martin Porter's own implementation in C, and had several vestigial quirks lying around. These include the claim that the stem() method takes a "char pointer" as an argument (no such thing in Python) and the need to pass in start and end indexes between which stem() should read the word from the given char array. At some point in nltk's history, during or prior to the 2006 commit that added porter.py to the current Git repository: https://github.com/nltk/nltk/commit/edf46779567450ec36e64a0decd75bcd5984e092 this was "solved" by renaming Vivake's stem() method to stem_word() and creating a wrapper for it called stem() that conformed to the StemmerI interface. This was completely pointless; the right thing to do would've been to remove the unnecessary parts of Vivake's stem() method and thereby acheive conformity to StemmerI. This commit does this, but at the cost of breaking backwards compatibility for anyone who was using stem_word(word) instead of stem(word); those people will need to adjust their application code when updating to the latest version of NLTK. --- nltk/stem/porter.py | 46 ++++++++++++++++++++------------------------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/nltk/stem/porter.py b/nltk/stem/porter.py index 9052a04239..ba15357d07 100644 --- a/nltk/stem/porter.py +++ b/nltk/stem/porter.py @@ -529,29 +529,6 @@ def _step5(self, word): return word - def stem_word(self, p, i=0, j=None): - """ - Returns the stem of p, or, if i and j are given, the stem of p[i:j+1]. - """ - ## --NLTK-- - if j is None and i == 0: - word = p - else: - if j is None: - j = len(p) - 1 - word = p[i:j+1] - - if word in self.pool: - return self.pool[word] - - word = self._step1ab(word) - word = self._step1c(word) - word = self._step2(word) - word = self._step3(word) - word = self._step4(word) - word = self._step5(word) - return word - def _adjust_case(self, word, stem): lower = word.lower() @@ -583,10 +560,27 @@ def _adjust_case(self, word, stem): # ret = ret + separator # return ret - ## --NLTK-- - ## Define a stem() method that implements the StemmerI interface. def stem(self, word): - stem = self.stem_word(word.lower(), 0, len(word) - 1) + stem = word.lower() + + # --NLTK-- + if word in self.pool: + return self.pool[word] + + if len(word) <= 2: + return word # --DEPARTURE-- + # With this line, strings of length 1 or 2 don't go through the + # stemming process, although no mention is made of this in the + # published algorithm. Remove the line to match the published + # algorithm. + + stem = self._step1ab(stem) + stem = self._step1c(stem) + stem = self._step2(stem) + stem = self._step3(stem) + stem = self._step4(stem) + stem = self._step5(stem) + return self._adjust_case(word, stem) def __repr__(self): From 8b7ffe6d8cafb8f53b049440fdbb2f6da1436a7e Mon Sep 17 00:00:00 2001 From: Mark Amery Date: Sat, 9 Jan 2016 16:16:37 +0000 Subject: [PATCH 05/17] [PorterStemmer] Remove more commented out code. I don't even know where this came from. It's not part of the Vivake Gupta version at http://tartarus.org/~martin/PorterStemmer/python.txt Regardless, there's no reason for it to remain here. --- nltk/stem/porter.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/nltk/stem/porter.py b/nltk/stem/porter.py index ba15357d07..b920604812 100644 --- a/nltk/stem/porter.py +++ b/nltk/stem/porter.py @@ -541,25 +541,6 @@ def _adjust_case(self, word, stem): return ret - ## --NLTK-- - ## Don't use this procedure; we want to work with individual - ## tokens, instead. (commented out the following procedure) - #def stem(self, text): - # parts = re.split("(\W+)", text) - # numWords = (len(parts) + 1)/2 - # - # ret = "" - # for i in xrange(numWords): - # word = parts[2 * i] - # separator = "" - # if ((2 * i) + 1) < len(parts): - # separator = parts[(2 * i) + 1] - # - # stem = self.stem_word(string.lower(word), 0, len(word) - 1) - # ret = ret + self.adjust_case(word, stem) - # ret = ret + separator - # return ret - def stem(self, word): stem = word.lower() From a2d1dfa7351f47f10f72108a118be227808c00ce Mon Sep 17 00:00:00 2001 From: Mark Amery Date: Sat, 9 Jan 2016 16:21:08 +0000 Subject: [PATCH 06/17] [PorterStemmer] Remove _adjust_case for consistency with Lancaster and Snowball Handling of upper and lower case is not specified in Martin Porter's "An algorithm for suffix stripping" paper; the algorithm description there never even mentions the existence of difference letter cases. Nor does Martin's C implementation of the stemmer at: http://tartarus.org/~martin/PorterStemmer/c.txt handle case in the way that NLTK's version has been doing; instead, it simply requires that the user convert their word to lowercase before calling stem(). Since there is no Porter-specific reason to preserve our (odd) behaviour here, and our other StemmerI implementations don't do it, we should probably purge it, as this commit does. --- nltk/stem/porter.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/nltk/stem/porter.py b/nltk/stem/porter.py index b920604812..cab74e1708 100644 --- a/nltk/stem/porter.py +++ b/nltk/stem/porter.py @@ -529,18 +529,6 @@ def _step5(self, word): return word - def _adjust_case(self, word, stem): - lower = word.lower() - - ret = "" - for x in range(len(stem)): - if lower[x] == stem[x]: - ret += word[x] - else: - ret += stem[x] - - return ret - def stem(self, word): stem = word.lower() @@ -562,7 +550,7 @@ def stem(self, word): stem = self._step4(stem) stem = self._step5(stem) - return self._adjust_case(word, stem) + return stem def __repr__(self): return '' From 829b29ac36918983d7d7e150b5beebc36c7268b4 Mon Sep 17 00:00:00 2001 From: Mark Amery Date: Sun, 10 Jan 2016 18:41:53 +0000 Subject: [PATCH 07/17] [PorterStemmer] Reimplement steps 1a and 1b while: - Making the code more readable and including quotes in comments that allow the code to be matched up with Porter's published algorithm - Marking NLTK-specific departure points properly --- nltk/stem/porter.py | 284 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 218 insertions(+), 66 deletions(-) diff --git a/nltk/stem/porter.py b/nltk/stem/porter.py index cab74e1708..f4c89412f2 100644 --- a/nltk/stem/porter.py +++ b/nltk/stem/porter.py @@ -59,6 +59,9 @@ from nltk.stem.api import StemmerI from nltk.compat import python_2_unicode_compatible +class _CannotReplaceSuffix(Exception): + pass + @python_2_unicode_compatible class PorterStemmer(StemmerI): @@ -174,6 +177,57 @@ def _m(self, word, j): break i = i + 1 i = i + 1 + + def _measure(self, stem): + """Returns the 'measure' of stem, per definition in the paper + + From the paper: + + A consonant will be denoted by c, a vowel by v. A list + ccc... of length greater than 0 will be denoted by C, and a + list vvv... of length greater than 0 will be denoted by V. + Any word, or part of a word, therefore has one of the four + forms: + + CVCV ... C + CVCV ... V + VCVC ... C + VCVC ... V + + These may all be represented by the single form + + [C]VCVC ... [V] + + where the square brackets denote arbitrary presence of their + contents. Using (VC){m} to denote VC repeated m times, this + may again be written as + + [C](VC){m}[V]. + + m will be called the \measure\ of any word or word part when + represented in this form. The case m = 0 covers the null + word. Here are some examples: + + m=0 TR, EE, TREE, Y, BY. + m=1 TROUBLE, OATS, TREES, IVY. + m=2 TROUBLES, PRIVATE, OATEN, ORRERY. + """ + cv_sequence = '' + + # Construct a string of 'c's and 'v's representing whether each + # character in `stem` is a consonsant or a vowel. + # e.g. 'falafel' becomes 'cvcvcvc', + # 'architecture' becomes 'vcccvcvccvcv' + for i in range(len(stem)): + if self._cons(stem, i): + cv_sequence += 'c' + else: + cv_sequence += 'v' + + # Count the number of 'vc' occurences, which is equivalent to + # the number of 'VC' occurrences in Porter's reduced form in the + # docstring above, which is in turn equivalent to `m` + return cv_sequence.count('vc') def _vowelinstem(self, stem): """vowelinstem(stem) is TRUE <=> stem contains a vowel""" @@ -190,6 +244,22 @@ def _doublec(self, word): return False return self._cons(word, len(word)-1) + def _ends_cvc(self, word): + """Implements condition *o from the paper + + From the paper: + + *o - the stem ends cvc, where the second c is not W, X or Y + (e.g. -WIL, -HOP). + """ + return ( + len(word) >= 3 and + self._cons(word, len(word) - 3) and + not self._cons(word, len(word) - 2) and + self._cons(word, len(word) - 1) and + word[-1] not in ('w', 'x', 'y') + ) + def _cvc(self, word, i): """cvc(i) is TRUE <=> @@ -212,74 +282,155 @@ def _cvc(self, word, i): return False return True + + def _replace_suffix(self, word, suffix, replacement): + """Replaces `suffix` of `word` with `replacement""" + assert word.endswith(suffix), "Given word doesn't end with given suffix" + return word[:-len(suffix)] + replacement - def _step1ab(self, word): - """step1ab() gets rid of plurals and -ed or -ing. e.g. - - caresses -> caress - ponies -> poni - sties -> sti - tie -> tie (--NEW--: see below) - caress -> caress - cats -> cat - - feed -> feed - agreed -> agree - disabled -> disable - - matting -> mat - mating -> mate - meeting -> meet - milling -> mill - messing -> mess - - meetings -> meet + def _replace_suffix_if(self, word, suffix, replacement, condition): + """If `condition`, replace suffix with replacement, else raise + + `condition` should be a lambda that takes the word and stem as + arguments and returns True or False. """ - if word[-1] == 's': - if word.endswith("sses"): - word = word[:-2] - elif word.endswith("ies"): - if len(word) == 4: - word = word[:-1] - # this line extends the original algorithm, so that - # 'flies'->'fli' but 'dies'->'die' etc - else: - word = word[:-2] - elif word[-2] != 's': - word = word[:-1] - - ed_or_ing_trimmed = False - if word.endswith("ied"): - if len(word) == 4: - word = word[:-1] + if not word.endswith(suffix): + raise _CannotReplaceSuffix("word does not end with suffix") + else: + stem = self._replace_suffix(word, suffix, replacement) + if condition is None or condition(stem): + return stem else: - word = word[:-2] - # this line extends the original algorithm, so that - # 'spied'->'spi' but 'died'->'die' etc - - elif word.endswith("eed"): - if self._m(word, len(word)-4) > 0: - word = word[:-1] - - - elif word.endswith("ed") and self._vowelinstem(word[:-2]): - word = word[:-2] - ed_or_ing_trimmed = True - elif word.endswith("ing") and self._vowelinstem(word[:-3]): - word = word[:-3] - ed_or_ing_trimmed = True - - if ed_or_ing_trimmed: - if word.endswith("at") or word.endswith("bl") or word.endswith("iz"): - word += 'e' - elif self._doublec(word): - if word[-1] not in ['l', 's', 'z']: - word = word[:-1] - elif (self._m(word, len(word)-1) == 1 and self._cvc(word, len(word)-1)): - word += 'e' - + raise _CannotReplaceSuffix("condition not met") + + def _apply_first_possible_rule(self, word, rules): + """Applies the first applicable suffix-removal rule to the word + + Takes a word and a list of suffix-removal rules represented as + 3-tuples, with the first element being the suffix to remove, + the second element being the string to replace it with, and the + final element being the condition for the rule to be applicable, + or None if the rule is unconditional. + """ + for rule in rules: + try: + return self._replace_suffix_if(word, *rule) + except _CannotReplaceSuffix: + pass + return word - + + def _step1a(self, word): + """Implements Step 1a from "An algorithm for suffix stripping" + + From the paper: + + SSES -> SS caresses -> caress + IES -> I ponies -> poni + ties -> ti + SS -> SS caress -> caress + S -> cats -> cat + """ + return self._apply_first_possible_rule(word, [ + ('sses', 'ss', None), # SSES -> SS + + # --NLTK-- + # this line extends the original algorithm, so that + # 'flies'->'fli' but 'dies'->'die' etc + ('ies', 'ie', lambda stem: len(word) == 4), + + ('ies', 'i', None), # IES -> I + ('ss', 'ss', None), # SS -> SS + ('s', '', None), # S -> + ]) + + def _step1b(self, word): + """Implements Step 1b from "An algorithm for suffix stripping" + + From the paper: + + (m>0) EED -> EE feed -> feed + agreed -> agree + (*v*) ED -> plastered -> plaster + bled -> bled + (*v*) ING -> motoring -> motor + sing -> sing + + If the second or third of the rules in Step 1b is successful, the following + is done: + + AT -> ATE conflat(ed) -> conflate + BL -> BLE troubl(ed) -> trouble + IZ -> IZE siz(ed) -> size + (*d and not (*L or *S or *Z)) + -> single letter + hopp(ing) -> hop + tann(ed) -> tan + fall(ing) -> fall + hiss(ing) -> hiss + fizz(ed) -> fizz + (m=1 and *o) -> E fail(ing) -> fail + fil(ing) -> file + + The rule to map to a single letter causes the removal of one of the double + letter pair. The -E is put back on -AT, -BL and -IZ, so that the suffixes + -ATE, -BLE and -IZE can be recognised later. This E may be removed in step + 4. + """ + # --NLTK-- + # this block extends the original algorithm, so that + # 'spied'->'spi' but 'died'->'die' etc + try: + return self._replace_suffix_if( + word, 'ied', 'ie', lambda stem: len(word) == 4 + ) + except _CannotReplaceSuffix: + pass + + try: + # (m>0) EED -> EE + return self._replace_suffix_if( + word, 'eed', 'ee', lambda stem: self._measure(stem) > 0 + ) + except _CannotReplaceSuffix: + pass + + rule_2_or_3_succeeded = False + for rule in [ + ('ed', '', self._vowelinstem), # (*v*) ED -> + ('ing', '', self._vowelinstem), # (*v*) ING -> + ]: + try: + intermediate_stem = self._replace_suffix_if(word, *rule) + rule_2_or_3_succeeded = True + break + except _CannotReplaceSuffix: + pass + + if not rule_2_or_3_succeeded: + return word + + final_letter = intermediate_stem[-1] + return self._apply_first_possible_rule(intermediate_stem, [ + ('at', 'ate', None), # AT -> ATE + ('bl', 'ble', None), # BL -> BLE + ('iz', 'ize', None), # IZ -> IZE + # (*d and not (*L or *S or *Z)) + # -> single letter + ( + final_letter*2, + final_letter, + lambda stem: final_letter not in ('l', 's', 'z') + ), + # (m=1 and *o) -> E + ( + '', + 'e', + lambda stem: (self._measure(stem) == 1 and + self._ends_cvc(stem)) + ), + ]) + def _step1c(self, word): """step1c() turns terminal y to i when there is another vowel in the stem. --NEW--: This has been modified from the original Porter algorithm so that y->i @@ -542,8 +693,9 @@ def stem(self, word): # stemming process, although no mention is made of this in the # published algorithm. Remove the line to match the published # algorithm. - - stem = self._step1ab(stem) + + stem = self._step1a(stem) + stem = self._step1b(stem) stem = self._step1c(stem) stem = self._step2(stem) stem = self._step3(stem) From 221c19375ce6b515b4384b8aadf79fc434c8cc19 Mon Sep 17 00:00:00 2001 From: Mark Amery Date: Sun, 10 Jan 2016 18:47:48 +0000 Subject: [PATCH 08/17] [PorterStemmer] Rename vowelinstem to use underscores for PEP 0008 compliance --- nltk/stem/porter.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nltk/stem/porter.py b/nltk/stem/porter.py index f4c89412f2..a61ff19d0b 100644 --- a/nltk/stem/porter.py +++ b/nltk/stem/porter.py @@ -229,8 +229,8 @@ def _measure(self, stem): # docstring above, which is in turn equivalent to `m` return cv_sequence.count('vc') - def _vowelinstem(self, stem): - """vowelinstem(stem) is TRUE <=> stem contains a vowel""" + def _contains_vowel(self, stem): + """_contains_vowel(stem) is TRUE <=> stem contains a vowel""" for i in range(len(stem)): if not self._cons(stem, i): return True @@ -397,8 +397,8 @@ def _step1b(self, word): rule_2_or_3_succeeded = False for rule in [ - ('ed', '', self._vowelinstem), # (*v*) ED -> - ('ing', '', self._vowelinstem), # (*v*) ING -> + ('ed', '', self._contains_vowel), # (*v*) ED -> + ('ing', '', self._contains_vowel), # (*v*) ING -> ]: try: intermediate_stem = self._replace_suffix_if(word, *rule) @@ -446,7 +446,7 @@ def _step1c(self, word): 'enjoy'. Step 1c is perhaps done too soon; but with this modification that no longer really matters. - Also, the removal of the vowelinstem(z) condition means that 'spy', 'fly', + Also, the removal of the contains_vowel(z) condition means that 'spy', 'fly', 'try' ... stem to 'spi', 'fli', 'tri' and conflate with 'spied', 'tried', 'flies' ... """ From ccd5525351b3c0a45b1ecea4880e2231a456e1ef Mon Sep 17 00:00:00 2001 From: Mark Amery Date: Sun, 10 Jan 2016 19:01:40 +0000 Subject: [PATCH 09/17] [PorterStemmer] Refactor step1c to new style. --- nltk/stem/porter.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/nltk/stem/porter.py b/nltk/stem/porter.py index a61ff19d0b..2ae100c2d2 100644 --- a/nltk/stem/porter.py +++ b/nltk/stem/porter.py @@ -432,7 +432,15 @@ def _step1b(self, word): ]) def _step1c(self, word): - """step1c() turns terminal y to i when there is another vowel in the stem. + """Implements Step 1c from "An algorithm for suffix stripping" + + From the paper: + + Step 1c + + (*v*) Y -> I happy -> happi + sky -> sky + --NEW--: This has been modified from the original Porter algorithm so that y->i is only done when y is preceded by a consonant, but not if the stem is only a single consonant, i.e. @@ -450,9 +458,14 @@ def _step1c(self, word): 'try' ... stem to 'spi', 'fli', 'tri' and conflate with 'spied', 'tried', 'flies' ... """ - if word[-1] == 'y' and len(word) > 2 and self._cons(word, len(word) - 2): - return word[:-1] + 'i' - else: + try: + return self._replace_suffix_if( + word, + 'y', + 'i', + lambda stem: len(word) > 2 and self._cons(word, len(word) - 2) + ) + except _CannotReplaceSuffix: return word def _step2(self, word): From 10498f87c200713785ac49a5955b511e57d14f25 Mon Sep 17 00:00:00 2001 From: Mark Amery Date: Sun, 10 Jan 2016 19:56:55 +0000 Subject: [PATCH 10/17] [PorterStemmer] Refactor step2 --- nltk/stem/porter.py | 164 ++++++++++++++++++++------------------------ 1 file changed, 75 insertions(+), 89 deletions(-) diff --git a/nltk/stem/porter.py b/nltk/stem/porter.py index 2ae100c2d2..3bf703cfca 100644 --- a/nltk/stem/porter.py +++ b/nltk/stem/porter.py @@ -469,96 +469,82 @@ def _step1c(self, word): return word def _step2(self, word): - """step2() maps double suffices to single ones. - so -ization ( = -ize plus -ation) maps to -ize etc. note that the - string before the suffix must give m() > 0. + """Implements Step 2 from "An algorithm for suffix stripping" + + From the paper: + + Step 2 + + (m>0) ATIONAL -> ATE relational -> relate + (m>0) TIONAL -> TION conditional -> condition + rational -> rational + (m>0) ENCI -> ENCE valenci -> valence + (m>0) ANCI -> ANCE hesitanci -> hesitance + (m>0) IZER -> IZE digitizer -> digitize + (m>0) ABLI -> ABLE conformabli -> conformable + (m>0) ALLI -> AL radicalli -> radical + (m>0) ENTLI -> ENT differentli -> different + (m>0) ELI -> E vileli - > vile + (m>0) OUSLI -> OUS analogousli -> analogous + (m>0) IZATION -> IZE vietnamization -> vietnamize + (m>0) ATION -> ATE predication -> predicate + (m>0) ATOR -> ATE operator -> operate + (m>0) ALISM -> AL feudalism -> feudal + (m>0) IVENESS -> IVE decisiveness -> decisive + (m>0) FULNESS -> FUL hopefulness -> hopeful + (m>0) OUSNESS -> OUS callousness -> callous + (m>0) ALITI -> AL formaliti -> formal + (m>0) IVITI -> IVE sensitiviti -> sensitive + (m>0) BILITI -> BLE sensibiliti -> sensible """ - if len(word) <= 1: # Only possible at this stage given unusual inputs to stem_word like 'oed' - return word - - ch = word[-2] - - if ch == 'a': - if word.endswith("ational"): - return word[:-7] + "ate" if self._m(word, len(word)-8) > 0 else word - elif word.endswith("tional"): - return word[:-2] if self._m(word, len(word)-7) > 0 else word - else: - return word - elif ch == 'c': - if word.endswith("enci"): - return word[:-4] + "ence" if self._m(word, len(word)-5) > 0 else word - elif word.endswith("anci"): - return word[:-4] + "ance" if self._m(word, len(word)-5) > 0 else word - else: - return word - elif ch == 'e': - if word.endswith("izer"): - return word[:-1] if self._m(word, len(word)-5) > 0 else word - else: - return word - elif ch == 'l': - if word.endswith("bli"): - return word[:-3] + "ble" if self._m(word, len(word)-4) > 0 else word # --DEPARTURE-- - # To match the published algorithm, replace "bli" with "abli" and "ble" with "able" - elif word.endswith("alli"): - # --NEW-- - if self._m(word, len(word)-5) > 0: - word = word[:-2] - return self._step2(word) - else: - return word - elif word.endswith("fulli"): - return word[:-2] if self._m(word, len(word)-6) else word # --NEW-- - elif word.endswith("entli"): - return word[:-2] if self._m(word, len(word)-6) else word - elif word.endswith("eli"): - return word[:-2] if self._m(word, len(word)-4) else word - elif word.endswith("ousli"): - return word[:-2] if self._m(word, len(word)-6) else word - else: - return word - elif ch == 'o': - if word.endswith("ization"): - return word[:-7] + "ize" if self._m(word, len(word)-8) else word - elif word.endswith("ation"): - return word[:-5] + "ate" if self._m(word, len(word)-6) else word - elif word.endswith("ator"): - return word[:-4] + "ate" if self._m(word, len(word)-5) else word - else: - return word - elif ch == 's': - if word.endswith("alism"): - return word[:-3] if self._m(word, len(word)-6) else word - elif word.endswith("ness"): - if word.endswith("iveness"): - return word[:-4] if self._m(word, len(word)-8) else word - elif word.endswith("fulness"): - return word[:-4] if self._m(word, len(word)-8) else word - elif word.endswith("ousness"): - return word[:-4] if self._m(word, len(word)-8) else word - else: - return word - else: - return word - elif ch == 't': - if word.endswith("aliti"): - return word[:-3] if self._m(word, len(word)-6) else word - elif word.endswith("iviti"): - return word[:-5] + "ive" if self._m(word, len(word)-6) else word - elif word.endswith("biliti"): - return word[:-6] + "ble" if self._m(word, len(word)-7) else word - else: - return word - elif ch == 'g': # --DEPARTURE-- - if word.endswith("logi"): - return word[:-1] if self._m(word, len(word) - 4) else word # --NEW-- (Barry Wilkins) - # To match the published algorithm, pass len(word)-5 to _m instead of len(word)-4 - else: - return word - - else: - return word + positive_measure = lambda stem: self._measure(stem) > 0 + + # --NEW-- + # Instead of applying the ALLI -> AL rule after 'bli' per the + # published algorithm, instead we apply it first, and, if it + # succeeds, run the result through step2 again. + try: + stem = self._replace_suffix_if(word, 'alli', 'al', positive_measure) + return self._step2(stem) + except _CannotReplaceSuffix: + pass + + return self._apply_first_possible_rule(word, [ + ('ational', 'ate', positive_measure), + ('tional', 'tion', positive_measure), + ('enci', 'ence', positive_measure), + ('anci', 'ance', positive_measure), + ('izer', 'ize', positive_measure), + + # --DEPARTURE-- + # To match the published algorithm, replace "bli" with + # "abli" and "ble" with "able" + ('bli', 'ble', positive_measure), + + # -- NEW -- + ('fulli', 'ful', positive_measure), + + ('entli', 'ent', positive_measure), + ('eli', 'e', positive_measure), + ('ousli', 'ous', positive_measure), + ('ization', 'ize', positive_measure), + ('ation', 'ate', positive_measure), + ('ator', 'ate', positive_measure), + ('alism', 'al', positive_measure), + ('iveness', 'ive', positive_measure), + ('fulness', 'ful', positive_measure), + ('ousness', 'ous', positive_measure), + ('aliti', 'al', positive_measure), + ('iviti', 'ive', positive_measure), + ('biliti', 'ble', positive_measure), + + # --DEPARTURE-- + # To match the published algorithm, delete this phrase + # --NEW-- (Barry Wilkins) + # To match the published algorithm, replace lambda below + # with just positive_measure + ("logi", "log", lambda stem: positive_measure(word[:-3])), + ]) def _step3(self, word): """step3() deals with -ic-, -full, -ness etc. similar strategy to step2.""" From 92d845f5a24ab6e9ec2bcaa190bf16f0fb837c1e Mon Sep 17 00:00:00 2001 From: Mark Amery Date: Sun, 10 Jan 2016 20:04:55 +0000 Subject: [PATCH 11/17] [PorterStemmer] Refactor step 3 --- nltk/stem/porter.py | 113 +++++++++++++++++++++----------------------- 1 file changed, 55 insertions(+), 58 deletions(-) diff --git a/nltk/stem/porter.py b/nltk/stem/porter.py index 3bf703cfca..d5e34cd95a 100644 --- a/nltk/stem/porter.py +++ b/nltk/stem/porter.py @@ -228,6 +228,9 @@ def _measure(self, stem): # the number of 'VC' occurrences in Porter's reduced form in the # docstring above, which is in turn equivalent to `m` return cv_sequence.count('vc') + + def _has_positive_measure(self, stem): + return self._measure(stem) > 0 def _contains_vowel(self, stem): """_contains_vowel(stem) is TRUE <=> stem contains a vowel""" @@ -497,89 +500,83 @@ def _step2(self, word): (m>0) IVITI -> IVE sensitiviti -> sensitive (m>0) BILITI -> BLE sensibiliti -> sensible """ - positive_measure = lambda stem: self._measure(stem) > 0 - + # --NEW-- # Instead of applying the ALLI -> AL rule after 'bli' per the # published algorithm, instead we apply it first, and, if it # succeeds, run the result through step2 again. try: - stem = self._replace_suffix_if(word, 'alli', 'al', positive_measure) + stem = self._replace_suffix_if( + word, + 'alli', + 'al', + self._has_positive_measure + ) return self._step2(stem) except _CannotReplaceSuffix: pass return self._apply_first_possible_rule(word, [ - ('ational', 'ate', positive_measure), - ('tional', 'tion', positive_measure), - ('enci', 'ence', positive_measure), - ('anci', 'ance', positive_measure), - ('izer', 'ize', positive_measure), + ('ational', 'ate', self._has_positive_measure), + ('tional', 'tion', self._has_positive_measure), + ('enci', 'ence', self._has_positive_measure), + ('anci', 'ance', self._has_positive_measure), + ('izer', 'ize', self._has_positive_measure), # --DEPARTURE-- # To match the published algorithm, replace "bli" with # "abli" and "ble" with "able" - ('bli', 'ble', positive_measure), + ('bli', 'ble', self._has_positive_measure), # -- NEW -- - ('fulli', 'ful', positive_measure), + ('fulli', 'ful', self._has_positive_measure), - ('entli', 'ent', positive_measure), - ('eli', 'e', positive_measure), - ('ousli', 'ous', positive_measure), - ('ization', 'ize', positive_measure), - ('ation', 'ate', positive_measure), - ('ator', 'ate', positive_measure), - ('alism', 'al', positive_measure), - ('iveness', 'ive', positive_measure), - ('fulness', 'ful', positive_measure), - ('ousness', 'ous', positive_measure), - ('aliti', 'al', positive_measure), - ('iviti', 'ive', positive_measure), - ('biliti', 'ble', positive_measure), + ('entli', 'ent', self._has_positive_measure), + ('eli', 'e', self._has_positive_measure), + ('ousli', 'ous', self._has_positive_measure), + ('ization', 'ize', self._has_positive_measure), + ('ation', 'ate', self._has_positive_measure), + ('ator', 'ate', self._has_positive_measure), + ('alism', 'al', self._has_positive_measure), + ('iveness', 'ive', self._has_positive_measure), + ('fulness', 'ful', self._has_positive_measure), + ('ousness', 'ous', self._has_positive_measure), + ('aliti', 'al', self._has_positive_measure), + ('iviti', 'ive', self._has_positive_measure), + ('biliti', 'ble', self._has_positive_measure), # --DEPARTURE-- # To match the published algorithm, delete this phrase # --NEW-- (Barry Wilkins) # To match the published algorithm, replace lambda below - # with just positive_measure - ("logi", "log", lambda stem: positive_measure(word[:-3])), + # with just self._has_positive_measure + ("logi", "log", lambda stem: self._has_positive_measure(word[:-3])), ]) def _step3(self, word): - """step3() deals with -ic-, -full, -ness etc. similar strategy to step2.""" - - ch = word[-1] - - if ch == 'e': - if word.endswith("icate"): - return word[:-3] if self._m(word, len(word)-6) else word - elif word.endswith("ative"): - return word[:-5] if self._m(word, len(word)-6) else word - elif word.endswith("alize"): - return word[:-3] if self._m(word, len(word)-6) else word - else: - return word - elif ch == 'i': - if word.endswith("iciti"): - return word[:-3] if self._m(word, len(word)-6) else word - else: - return word - elif ch == 'l': - if word.endswith("ical"): - return word[:-2] if self._m(word, len(word)-5) else word - elif word.endswith("ful"): - return word[:-3] if self._m(word, len(word)-4) else word - else: - return word - elif ch == 's': - if word.endswith("ness"): - return word[:-4] if self._m(word, len(word)-5) else word - else: - return word - - else: - return word + """Implements Step 3 from "An algorithm for suffix stripping" + + From the paper: + + Step 3 + + (m>0) ICATE -> IC triplicate -> triplic + (m>0) ATIVE -> formative -> form + (m>0) ALIZE -> AL formalize -> formal + (m>0) ICITI -> IC electriciti -> electric + (m>0) ICAL -> IC electrical -> electric + (m>0) FUL -> hopeful -> hope + (m>0) NESS -> goodness -> good + """ + return self._apply_first_possible_rule(word, [ + ('icate', 'ic', self._has_positive_measure), + ('ative', '', self._has_positive_measure), + ('alize', 'al', self._has_positive_measure), + ('iciti', 'ic', self._has_positive_measure), + ('ical', 'ic', self._has_positive_measure), + ('ful', '', self._has_positive_measure), + ('ness', '', self._has_positive_measure), + ]) def _step4(self, word): """step4() takes off -ant, -ence etc., in context vcvc.""" From 9ad2b28925cc3ca3ec7301fdb22026b470a586bf Mon Sep 17 00:00:00 2001 From: Mark Amery Date: Sun, 10 Jan 2016 20:15:56 +0000 Subject: [PATCH 12/17] [PorterStemmer] Refactor step 4 --- nltk/stem/porter.py | 140 ++++++++++++++++++-------------------------- 1 file changed, 57 insertions(+), 83 deletions(-) diff --git a/nltk/stem/porter.py b/nltk/stem/porter.py index d5e34cd95a..476fe698a2 100644 --- a/nltk/stem/porter.py +++ b/nltk/stem/porter.py @@ -579,89 +579,63 @@ def _step3(self, word): ]) def _step4(self, word): - """step4() takes off -ant, -ence etc., in context vcvc.""" - - if len(word) <= 1: # Only possible at this stage given unusual inputs to stem_word like 'oed' - return word - - ch = word[-2] - - if ch == 'a': - if word.endswith("al"): - return word[:-2] if self._m(word, len(word)-3) > 1 else word - else: - return word - elif ch == 'c': - if word.endswith("ance"): - return word[:-4] if self._m(word, len(word)-5) > 1 else word - elif word.endswith("ence"): - return word[:-4] if self._m(word, len(word)-5) > 1 else word - else: - return word - elif ch == 'e': - if word.endswith("er"): - return word[:-2] if self._m(word, len(word)-3) > 1 else word - else: - return word - elif ch == 'i': - if word.endswith("ic"): - return word[:-2] if self._m(word, len(word)-3) > 1 else word - else: - return word - elif ch == 'l': - if word.endswith("able"): - return word[:-4] if self._m(word, len(word)-5) > 1 else word - elif word.endswith("ible"): - return word[:-4] if self._m(word, len(word)-5) > 1 else word - else: - return word - elif ch == 'n': - if word.endswith("ant"): - return word[:-3] if self._m(word, len(word)-4) > 1 else word - elif word.endswith("ement"): - return word[:-5] if self._m(word, len(word)-6) > 1 else word - elif word.endswith("ment"): - return word[:-4] if self._m(word, len(word)-5) > 1 else word - elif word.endswith("ent"): - return word[:-3] if self._m(word, len(word)-4) > 1 else word - else: - return word - elif ch == 'o': - if word.endswith("sion") or word.endswith("tion"): # slightly different logic to all the other cases - return word[:-3] if self._m(word, len(word)-4) > 1 else word - elif word.endswith("ou"): - return word[:-2] if self._m(word, len(word)-3) > 1 else word - else: - return word - elif ch == 's': - if word.endswith("ism"): - return word[:-3] if self._m(word, len(word)-4) > 1 else word - else: - return word - elif ch == 't': - if word.endswith("ate"): - return word[:-3] if self._m(word, len(word)-4) > 1 else word - elif word.endswith("iti"): - return word[:-3] if self._m(word, len(word)-4) > 1 else word - else: - return word - elif ch == 'u': - if word.endswith("ous"): - return word[:-3] if self._m(word, len(word)-4) > 1 else word - else: - return word - elif ch == 'v': - if word.endswith("ive"): - return word[:-3] if self._m(word, len(word)-4) > 1 else word - else: - return word - elif ch == 'z': - if word.endswith("ize"): - return word[:-3] if self._m(word, len(word)-4) > 1 else word - else: - return word - else: - return word + """Implements Step 4 from "An algorithm for suffix stripping" + + Step 4 + + (m>1) AL -> revival -> reviv + (m>1) ANCE -> allowance -> allow + (m>1) ENCE -> inference -> infer + (m>1) ER -> airliner -> airlin + (m>1) IC -> gyroscopic -> gyroscop + (m>1) ABLE -> adjustable -> adjust + (m>1) IBLE -> defensible -> defens + (m>1) ANT -> irritant -> irrit + (m>1) EMENT -> replacement -> replac + (m>1) MENT -> adjustment -> adjust + (m>1) ENT -> dependent -> depend + (m>1 and (*S or *T)) ION -> adoption -> adopt + (m>1) OU -> homologou -> homolog + (m>1) ISM -> communism -> commun + (m>1) ATE -> activate -> activ + (m>1) ITI -> angulariti -> angular + (m>1) OUS -> homologous -> homolog + (m>1) IVE -> effective -> effect + (m>1) IZE -> bowdlerize -> bowdler + + The suffixes are now removed. All that remains is a little + tidying up. + """ + measure_gt_1 = lambda stem: self._measure(stem) > 1 + + return self._apply_first_possible_rule(word, [ + ('al', '', measure_gt_1), + ('ance', '', measure_gt_1), + ('ence', '', measure_gt_1), + ('er', '', measure_gt_1), + ('ic', '', measure_gt_1), + ('able', '', measure_gt_1), + ('ible', '', measure_gt_1), + ('ant', '', measure_gt_1), + ('ement', '', measure_gt_1), + ('ment', '', measure_gt_1), + ('ent', '', measure_gt_1), + + # (m>1 and (*S or *T)) ION -> + ( + 'ion', + '', + lambda stem: self._measure(stem) > 1 and stem[-1] in ('s', 't') + ), + + ('ou', '', measure_gt_1), + ('ism', '', measure_gt_1), + ('ate', '', measure_gt_1), + ('iti', '', measure_gt_1), + ('ous', '', measure_gt_1), + ('ive', '', measure_gt_1), + ('ize', '', measure_gt_1), + ]) def _step5(self, word): """step5() removes a final -e if m() > 1, and changes -ll to -l if From cb160de791a73fb882107eac6fdf51429ab0b4b0 Mon Sep 17 00:00:00 2001 From: Mark Amery Date: Sun, 10 Jan 2016 21:07:45 +0000 Subject: [PATCH 13/17] [PorterStemmer] Refactor step5 (as step5a and step5b) --- nltk/stem/porter.py | 53 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 42 insertions(+), 11 deletions(-) diff --git a/nltk/stem/porter.py b/nltk/stem/porter.py index 476fe698a2..241b3e1421 100644 --- a/nltk/stem/porter.py +++ b/nltk/stem/porter.py @@ -636,19 +636,49 @@ def _step4(self, word): ('ive', '', measure_gt_1), ('ize', '', measure_gt_1), ]) + + def _step5a(self, word): + """Implements Step 5a from "An algorithm for suffix stripping" + + From the paper: + + Step 5a - def _step5(self, word): - """step5() removes a final -e if m() > 1, and changes -ll to -l if - m() > 1. + (m>1) E -> probate -> probat + rate -> rate + (m=1 and not *o) E -> cease -> ceas """ - if word[-1] == 'e': - a = self._m(word, len(word)-1) - if a > 1 or (a == 1 and not self._cvc(word, len(word)-2)): - word = word[:-1] - if word.endswith('ll') and self._m(word, len(word)-1) > 1: - word = word[:-1] + return self._apply_first_possible_rule(word, [ + ('e', '', lambda stem: self._measure(stem) > 1), + ( + 'e', + '', + lambda stem: ( + self._measure(stem) == 1 and + not self._ends_cvc(stem) + ) + ) + ]) - return word + def _step5b(self, word): + """Implements Step 5a from "An algorithm for suffix stripping" + + From the paper: + + Step 5b + + (m > 1 and *d and *L) -> single letter + controll -> control + roll -> roll + """ + # The rule is expressed in an overcomplicated way in Porter's + # paper, but all it means it that double-l should become + # single-l. It could've been written more straightforwardly as: + # + # (m > 1) LL -> L + return self._apply_first_possible_rule(word, [ + ('ll', 'l', lambda stem: self._measure(stem) > 1) + ]) def stem(self, word): stem = word.lower() @@ -670,7 +700,8 @@ def stem(self, word): stem = self._step2(stem) stem = self._step3(stem) stem = self._step4(stem) - stem = self._step5(stem) + stem = self._step5a(stem) + stem = self._step5b(stem) return stem From fab16baf17a4aec305136ae43115c3f07936b5dc Mon Sep 17 00:00:00 2001 From: Mark Amery Date: Sun, 10 Jan 2016 21:09:30 +0000 Subject: [PATCH 14/17] [PorterStemmer] Purge unused private methods obsoleted by the refactor --- nltk/stem/porter.py | 72 +-------------------------------------------- 1 file changed, 1 insertion(+), 71 deletions(-) diff --git a/nltk/stem/porter.py b/nltk/stem/porter.py index 241b3e1421..7e6f91b4cb 100644 --- a/nltk/stem/porter.py +++ b/nltk/stem/porter.py @@ -138,46 +138,7 @@ def _cons(self, word, i): else: return (not self._cons(word, i - 1)) return True - - def _m(self, word, j): - """m() measures the number of consonant sequences between k0 and j. - if c is a consonant sequence and v a vowel sequence, and <..> - indicates arbitrary presence, - - gives 0 - vc gives 1 - vcvc gives 2 - vcvcvc gives 3 - .... - """ - n = 0 - i = 0 - while True: - if i > j: - return n - if not self._cons(word, i): - break - i = i + 1 - i = i + 1 - - while True: - while True: - if i > j: - return n - if self._cons(word, i): - break - i = i + 1 - i = i + 1 - n = n + 1 - - while True: - if i > j: - return n - if not self._cons(word, i): - break - i = i + 1 - i = i + 1 - + def _measure(self, stem): """Returns the 'measure' of stem, per definition in the paper @@ -239,14 +200,6 @@ def _contains_vowel(self, stem): return True return False - def _doublec(self, word): - """doublec(word) is TRUE <=> word ends with a double consonant""" - if len(word) < 2: - return False - if (word[-1] != word[-2]): - return False - return self._cons(word, len(word)-1) - def _ends_cvc(self, word): """Implements condition *o from the paper @@ -262,29 +215,6 @@ def _ends_cvc(self, word): self._cons(word, len(word) - 1) and word[-1] not in ('w', 'x', 'y') ) - - def _cvc(self, word, i): - """cvc(i) is TRUE <=> - - a) ( --NEW--) i == 1, and word[0] word[1] is vowel consonant, or - - b) word[i - 2], word[i - 1], word[i] has the form consonant - - vowel - consonant and also if the second c is not w, x or y. this - is used when trying to restore an e at the end of a short word. - e.g. - - cav(e), lov(e), hop(e), crim(e), but - snow, box, tray. - """ - if i == 0: return False # i == 0 never happens perhaps - if i == 1: return (not self._cons(word, 0) and self._cons(word, 1)) - if not self._cons(word, i) or self._cons(word, i-1) or not self._cons(word, i-2): return False - - ch = word[i] - if ch == 'w' or ch == 'x' or ch == 'y': - return False - - return True def _replace_suffix(self, word, suffix, replacement): """Replaces `suffix` of `word` with `replacement""" From dccf39671479724e0f703adad8bc2b444e7214f5 Mon Sep 17 00:00:00 2001 From: Mark Amery Date: Sun, 10 Jan 2016 21:10:47 +0000 Subject: [PATCH 15/17] [PorterStemmer] Fix some odd indentation --- nltk/stem/porter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nltk/stem/porter.py b/nltk/stem/porter.py index 7e6f91b4cb..b2f8e63857 100644 --- a/nltk/stem/porter.py +++ b/nltk/stem/porter.py @@ -119,7 +119,7 @@ def __init__(self): "proceed" : ["proceed"], "exceed" : ["exceed"], "succeed" : ["succeed"], # Hiranmay Ghosh - } + } self.pool = {} for key in irregular_forms: From e7ec0babff7063da0842ecc6bc0c110419d624c1 Mon Sep 17 00:00:00 2001 From: Mark Amery Date: Mon, 11 Jan 2016 00:08:52 +0000 Subject: [PATCH 16/17] [PorterStemmer] Create 'modes' to let users use a faithful version of the algorithm. Also sort out the docs. --- nltk/stem/porter.py | 326 +++++++++++++++++++++++--------------------- 1 file changed, 167 insertions(+), 159 deletions(-) diff --git a/nltk/stem/porter.py b/nltk/stem/porter.py index b2f8e63857..48e848f6c8 100644 --- a/nltk/stem/porter.py +++ b/nltk/stem/porter.py @@ -1,57 +1,25 @@ -# Additional modifications were made to incorporate this module into -# NLTK. All such modifications are marked with "--NLTK--". The NLTK -# version of this module is maintained by NLTK developers, -# and is available via http://nltk.org/ - """ Porter Stemmer -This is the Porter stemming algorithm, ported to Python from the -version coded up in ANSI C by the author. It follows the algorithm +This is the Porter stemming algorithm. It follows the algorithm presented in Porter, M. "An algorithm for suffix stripping." Program 14.3 (1980): 130-137. -only differing from it at the points marked --DEPARTURE-- and --NEW-- -below. +with some optional deviations that can be turned on or off with the +`mode` argument to the constructor. -For a more faithful version of the Porter algorithm, see +Martin Porter, the algorithm's inventor, maintains a web page about the +algorithm at http://www.tartarus.org/~martin/PorterStemmer/ -Later additions: - - June 2000 - - The 'l' of the 'logi' -> 'log' rule is put with the stem, so that - short stems like 'geo' 'theo' etc work like 'archaeo' 'philo' etc. - - This follows a suggestion of Barry Wilkins, research student at - Birmingham. - - - February 2000 - - the cvc test for not dropping final -e now looks after vc at the - beginning of a word, so are, eve, ice, ore, use keep final -e. In this - test c is any consonant, including w, x and y. This extension was - suggested by Chris Emerson. - - -fully -> -ful treated like -fulness -> -ful, and - -tionally -> -tion treated like -tional -> -tion - - both in Step 2. These were suggested by Hiranmay Ghosh, of New Delhi. - - Invariants proceed, succeed, exceed. Also suggested by Hiranmay Ghosh. - -Additional modifications were made to incorperate this module into -nltk. All such modifications are marked with \"--NLTK--\". +which includes another Python implementation and other implementations +in many languages. """ from __future__ import print_function, unicode_literals -## --NLTK-- -## Declare this module's documentation format. __docformat__ = 'plaintext' import re @@ -64,67 +32,85 @@ class _CannotReplaceSuffix(Exception): @python_2_unicode_compatible class PorterStemmer(StemmerI): - - ## --NLTK-- - ## Add a module docstring """ A word stemmer based on the Porter stemming algorithm. - Porter, M. \"An algorithm for suffix stripping.\" + Porter, M. "An algorithm for suffix stripping." Program 14.3 (1980): 130-137. - - A few minor modifications have been made to Porter's basic - algorithm. See the source code of this module for more - information. - - The Porter Stemmer requires that all tokens have string types. + + See http://www.tartarus.org/~martin/PorterStemmer/ for the homepage + of the algorithm. + + Martin Porter has endorsed several modifications to the Porter + algorithm since writing his original paper, and those extensions are + included in the implementations on his website. Additionally, others + have proposed further improvements to the algorithm, including NLTK + contributors. There are thus three modes that can be selected by + passing the appropriate constant to the class constructor's `mode` + attribute: + + PorterStemmer.ORIGINAL_ALGORITHM + - Implementation that is faithful to the original paper. + + PorterStemmer.MARTIN_EXTENSIONS + - Implementation that only uses the modifications to the + algorithm that are included in the implementations on Martin + Porter's website. He has declared Porter frozen, so the + behaviour of those implementations should never change. + + PorterStemmer.NLTK_EXTENSIONS (default) + - Implementation that includes further improvements devised by + NLTK contributors or taken from other modified implementations + found on the web. + + For the best stemming, you should use the default NLTK_EXTENSIONS + version. However, if you need to get the same results as either the + original algorithm or one of Martin Porter's hosted versions for + compability reasons, you can use one of the other modes instead. """ - - # The main part of the stemming algorithm starts here. - # Note that only lower case sequences are stemmed. Forcing to lower case - # should be done before stem(...) is called. - - def __init__(self): - - ## --NEW-- - ## This is a table of irregular forms. It is quite short, but still - ## reflects the errors actually drawn to Martin Porter's attention over - ## a 20 year period! - ## - ## Extend it as necessary. - ## - ## The form of the table is: - ## { - ## "p1" : ["s11","s12","s13", ... ], - ## "p2" : ["s21","s22","s23", ... ], - ## ... - ## "pn" : ["sn1","sn2","sn3", ... ] - ## } - ## - ## String sij is mapped to paradigm form pi, and the main stemming - ## process is then bypassed. - - irregular_forms = { - "sky" : ["sky", "skies"], - "die" : ["dying"], - "lie" : ["lying"], - "tie" : ["tying"], - "news" : ["news"], - "inning" : ["innings", "inning"], - "outing" : ["outings", "outing"], - "canning" : ["cannings", "canning"], - "howe" : ["howe"], - - # --NEW-- - "proceed" : ["proceed"], - "exceed" : ["exceed"], - "succeed" : ["succeed"], # Hiranmay Ghosh - } - - self.pool = {} - for key in irregular_forms: - for val in irregular_forms[key]: - self.pool[val] = key + + # Modes the Stemmer can be instantiated in + NLTK_EXTENSIONS = 'NLTK_EXTENSIONS' + MARTIN_EXTENSIONS = 'MARTIN_EXTENSIONS' + ORIGINAL_ALGORITHM = 'ORIGINAL_ALGORITHM' + + def __init__(self, mode=NLTK_EXTENSIONS): + if mode not in ( + self.NLTK_EXTENSIONS, + self.MARTIN_EXTENSIONS, + self.ORIGINAL_ALGORITHM + ): + raise ValueError( + "Mode must be one of PorterStemmer.NLTK_EXTENSIONS, " + "PorterStemmer.MARTIN_EXTENSIONS, or " + "PorterStemmer.ORIGINAL_ALGORITHM" + ) + + self.mode = mode + + if self.mode == self.NLTK_EXTENSIONS: + # This is a table of irregular forms. It is quite short, + # but still reflects the errors actually drawn to Martin + # Porter's attention over a 20 year period! + irregular_forms = { + "sky" : ["sky", "skies"], + "die" : ["dying"], + "lie" : ["lying"], + "tie" : ["tying"], + "news" : ["news"], + "inning" : ["innings", "inning"], + "outing" : ["outings", "outing"], + "canning" : ["cannings", "canning"], + "howe" : ["howe"], + "proceed" : ["proceed"], + "exceed" : ["exceed"], + "succeed" : ["succeed"], + } + + self.pool = {} + for key in irregular_forms: + for val in irregular_forms[key]: + self.pool[val] = key self.vowels = frozenset(['a', 'e', 'i', 'o', 'u']) @@ -267,10 +253,14 @@ def _step1a(self, word): return self._apply_first_possible_rule(word, [ ('sses', 'ss', None), # SSES -> SS - # --NLTK-- - # this line extends the original algorithm, so that - # 'flies'->'fli' but 'dies'->'die' etc - ('ies', 'ie', lambda stem: len(word) == 4), + # this NLTK-only rule extends the original algorithm, so + # that 'flies'->'fli' but 'dies'->'die' etc + ( + 'ies', + 'ie', + lambda stem: (self.mode == self.NLTK_EXTENSIONS and + len(word) == 4) + ), ('ies', 'i', None), # IES -> I ('ss', 'ss', None), # SS -> SS @@ -310,15 +300,15 @@ def _step1b(self, word): -ATE, -BLE and -IZE can be recognised later. This E may be removed in step 4. """ - # --NLTK-- - # this block extends the original algorithm, so that + # this NLTK-only block extends the original algorithm, so that # 'spied'->'spi' but 'died'->'die' etc - try: - return self._replace_suffix_if( - word, 'ied', 'ie', lambda stem: len(word) == 4 - ) - except _CannotReplaceSuffix: - pass + if self.mode == self.NLTK_EXTENSIONS: + try: + return self._replace_suffix_if( + word, 'ied', 'ie', lambda stem: len(word) == 4 + ) + except _CannotReplaceSuffix: + pass try: # (m>0) EED -> EE @@ -373,30 +363,38 @@ def _step1c(self, word): (*v*) Y -> I happy -> happi sky -> sky - - --NEW--: This has been modified from the original Porter algorithm so that y->i - is only done when y is preceded by a consonant, but not if the stem - is only a single consonant, i.e. + """ + def nltk_condition(stem): + """ + This has been modified from the original Porter algorithm so + that y->i is only done when y is preceded by a consonant, + but not if the stem is only a single consonant, i.e. - (*c and not c) Y -> I + (*c and not c) Y -> I - So 'happy' -> 'happi', but - 'enjoy' -> 'enjoy' etc + So 'happy' -> 'happi', but + 'enjoy' -> 'enjoy' etc - This is a much better rule. Formerly 'enjoy'->'enjoi' and 'enjoyment'-> - 'enjoy'. Step 1c is perhaps done too soon; but with this modification that - no longer really matters. + This is a much better rule. Formerly 'enjoy'->'enjoi' and + 'enjoyment'->'enjoy'. Step 1c is perhaps done too soon; but + with this modification that no longer really matters. - Also, the removal of the contains_vowel(z) condition means that 'spy', 'fly', - 'try' ... stem to 'spi', 'fli', 'tri' and conflate with 'spied', 'tried', - 'flies' ... - """ + Also, the removal of the contains_vowel(z) condition means + that 'spy', 'fly', 'try' ... stem to 'spi', 'fli', 'tri' and + conflate with 'spied', 'tried', 'flies' ... + """ + return len(stem) > 1 and self._cons(stem, len(stem) - 1) + + def original_condition(stem): + return self._contains_vowel(stem) + try: return self._replace_suffix_if( word, 'y', 'i', - lambda stem: len(word) > 2 and self._cons(word, len(word) - 2) + nltk_condition if self.mode == self.NLTK_EXTENSIONS + else original_condition ) except _CannotReplaceSuffix: return word @@ -431,36 +429,34 @@ def _step2(self, word): (m>0) BILITI -> BLE sensibiliti -> sensible """ - # --NEW-- - # Instead of applying the ALLI -> AL rule after 'bli' per the - # published algorithm, instead we apply it first, and, if it - # succeeds, run the result through step2 again. - try: - stem = self._replace_suffix_if( - word, - 'alli', - 'al', - self._has_positive_measure - ) - return self._step2(stem) - except _CannotReplaceSuffix: - pass + if self.mode == self.NLTK_EXTENSIONS: + # Instead of applying the ALLI -> AL rule after '(a)bli' per + # the published algorithm, instead we apply it first, and, + # if it succeeds, run the result through step2 again. + try: + stem = self._replace_suffix_if( + word, + 'alli', + 'al', + self._has_positive_measure + ) + return self._step2(stem) + except _CannotReplaceSuffix: + pass - return self._apply_first_possible_rule(word, [ + bli_rule = ('bli', 'ble', self._has_positive_measure) + abli_rule = ('abli', 'able', self._has_positive_measure) + + rules = [ ('ational', 'ate', self._has_positive_measure), ('tional', 'tion', self._has_positive_measure), ('enci', 'ence', self._has_positive_measure), ('anci', 'ance', self._has_positive_measure), ('izer', 'ize', self._has_positive_measure), - # --DEPARTURE-- - # To match the published algorithm, replace "bli" with - # "abli" and "ble" with "able" - ('bli', 'ble', self._has_positive_measure), - - # -- NEW -- - ('fulli', 'ful', self._has_positive_measure), + abli_rule if self.mode == self.ORIGINAL_ALGORITHM else bli_rule, + ('alli', 'al', self._has_positive_measure), ('entli', 'ent', self._has_positive_measure), ('eli', 'e', self._has_positive_measure), ('ousli', 'ous', self._has_positive_measure), @@ -474,14 +470,28 @@ def _step2(self, word): ('aliti', 'al', self._has_positive_measure), ('iviti', 'ive', self._has_positive_measure), ('biliti', 'ble', self._has_positive_measure), + ] + + if self.mode == self.NLTK_EXTENSIONS: + rules.append( + ('fulli', 'ful', self._has_positive_measure) + ) - # --DEPARTURE-- - # To match the published algorithm, delete this phrase - # --NEW-- (Barry Wilkins) - # To match the published algorithm, replace lambda below - # with just self._has_positive_measure - ("logi", "log", lambda stem: self._has_positive_measure(word[:-3])), - ]) + # The 'l' of the 'logi' -> 'log' rule is put with the stem, + # so that short stems like 'geo' 'theo' etc work like + # 'archaeo' 'philo' etc. + rules.append(( + "logi", + "log", + lambda stem: self._has_positive_measure(word[:-3]) + )) + + if self.mode == self.MARTIN_EXTENSIONS: + rules.append( + ("logi", "log", self._has_positive_measure) + ) + + return self._apply_first_possible_rule(word, rules) def _step3(self, word): """Implements Step 3 from "An algorithm for suffix stripping" @@ -613,16 +623,14 @@ def _step5b(self, word): def stem(self, word): stem = word.lower() - # --NLTK-- - if word in self.pool: + if self.mode == self.NLTK_EXTENSIONS and word in self.pool: return self.pool[word] - if len(word) <= 2: - return word # --DEPARTURE-- - # With this line, strings of length 1 or 2 don't go through the - # stemming process, although no mention is made of this in the - # published algorithm. Remove the line to match the published - # algorithm. + if self.mode != self.ORIGINAL_ALGORITHM and len(word) <= 2: + # With this line, strings of length 1 or 2 don't go through + # the stemming process, although no mention is made of this + # in the published algorithm. + return word stem = self._step1a(stem) stem = self._step1b(stem) From 0cc39c6f525b9bf53eca9d7709e082201529cae4 Mon Sep 17 00:00:00 2001 From: Mark Amery Date: Mon, 11 Jan 2016 00:16:34 +0000 Subject: [PATCH 17/17] [PorterStemmer] Mention the contributions of people whose work we've used in the AUTHORS file --- AUTHORS.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/AUTHORS.md b/AUTHORS.md index 050b40f8f8..dbf553b796 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -189,3 +189,11 @@ - Sergio Oller - Will Monroe - Elijah Rippeth + +## Others whose work we've taken and included in NLTK, but who didn't directly contribute it: +### Contributors to the Porter Stemmer +- Martin Porter +- Vivake Gupta +- Barry Wilkins +- Hiranmay Ghosh +- Chris Emerson \ No newline at end of file