app/django/utils/stopwords.py

   1 # Performance note: I benchmarked this code using a set instead of
   2 # a list for the stopwords and was surprised to find that the list
   3 # performed /better/ than the set - maybe because it's only a small
   4 # list.
   5
   6 stopwords = '''
   7 i
   8 a
   9 an
  10 are
  11 as
  12 at
  13 be
  14 by
  15 for
  16 from
  17 how
  18 in
  19 is
  20 it
  21 of
  22 on
  23 or
  24 that
  25 the
  26 this
  27 to
  28 was
  29 what
  30 when
  31 where
  32 '''.split()
  33
  34 def strip_stopwords(sentence):
  35     "Removes stopwords - also normalizes whitespace"
  36     words = sentence.split()
  37     sentence = []
  38     for word in words:
  39         if word.lower() not in stopwords:
  40             sentence.append(word)
  41     return u' '.join(sentence)
  42