src/Gargantext/Core/Text/Terms/Mono/Stem.hs

   1 {-|
   2 Module      : Gargantext.Core.Text.Ngrams.Stem
   3 Description :
   4 Copyright   : (c) CNRS, 2017-Present
   5 License     : AGPL + CECILL v3
   6 Maintainer  : team@gargantext.org
   7 Stability   : experimental
   8 Portability : POSIX
   9
  10 In linguistic morphology and information retrieval, stemming is the
  11 process of reducing inflected (or sometimes derived) words to their word
  12 stem, base or root form—generally a written word form. The @stem@ needs
  13 not be identical to the morphological root of the word; it is usually
  14 sufficient that related words map to the same stem, even if this stem is
  15 not in itself a valid root.
  16 Source : https://en.wikipedia.org/wiki/Stemming
  17
  18 -}
  19
  20
  21 module Gargantext.Core.Text.Terms.Mono.Stem (stem, Lang(..))
  22   where
  23
  24 import Data.Text (Text)
  25 import qualified Data.Text   as DT
  26 import qualified NLP.Stemmer as N
  27
  28 import Gargantext.Prelude
  29 import Gargantext.Core (Lang(..))
  30
  31 -- (stem, Stemmer(..))
  32
  33 --import Language.Aspell (check, suggest, spellChecker, spellCheckerWithOptions)
  34 --import Language.Aspell.Options (ACOption(..))
  35
  36
  37 -- | Stemmer
  38
  39 -- A stemmer for English, for example, should identify the string "cats"
  40 -- (and possibly "catlike", "catty" etc.) as based on the root "cat".
  41
  42 -- and
  43 -- "stems", "stemmer", "stemming", "stemmed" as based on "stem". A stemming
  44 -- algorithm reduces the words "fishing", "fished", and "fisher" to the
  45 -- root word, "fish". On the other hand, "argue", "argued", "argues",
  46 -- "arguing", and "argus" reduce to the stem "argu" (illustrating the
  47 -- case where the stem is not itself a word or root) but "argument" and
  48 -- "arguments" reduce to the stem "argument".
  49
  50
  51 stem :: Lang -> Text -> Text
  52 stem lang = DT.pack . N.stem lang' . DT.unpack
  53   where
  54     lang' = case lang of
  55               EN -> N.English
  56               FR -> N.French
  57               _  -> panic $ DT.pack "not implemented yet"
  58
  59
  60