src/Gargantext/Text/Terms/Mono.hs

   1 {-|
   2 Module      : Gargantext.Text.Terms.Mono
   3 Description : Mono Terms module
   4 Copyright   : (c) CNRS, 2017 - present
   5 License     : AGPL + CECILL v3
   6 Maintainer  : team@gargantext.org
   7 Stability   : experimental
   8 Portability : POSIX
   9
  10 Mono-terms are Nterms where n == 1.
  11
  12 -}
  13
  14 {-# LANGUAGE NoImplicitPrelude #-}
  15
  16 module Gargantext.Text.Terms.Mono (monoTerms, monoTexts, monoTextsBySentence)
  17   where
  18
  19 import Prelude (String)
  20
  21 import Data.Text (Text)
  22 import qualified Data.Text as T
  23
  24 import qualified Data.List as L
  25 import qualified Data.Set as S
  26
  27 import Gargantext.Core
  28 import Gargantext.Core.Types
  29 import Gargantext.Text.Terms.Mono.Stem (stem)
  30
  31 import Gargantext.Prelude
  32 --import Data.Char (isAlphaNum, isSpace)
  33
  34 -- | TODO remove Num ?
  35 --isGram  c  = isAlphaNum c
  36
  37
  38 -- | Sentence split separators
  39 isSep :: Char -> Bool
  40 isSep = (`elem` (",.:;?!(){}[]\"" :: String))
  41
  42 monoTerms :: Lang -> Text -> [Terms]
  43 monoTerms l txt = map (monoText2term l) $ monoTexts txt
  44
  45 monoTexts :: Text -> [Text]
  46 monoTexts = L.concat . monoTextsBySentence
  47
  48 monoText2term :: Lang -> Text -> Terms
  49 monoText2term lang txt = Terms [txt] (S.singleton $ stem lang txt)
  50
  51 monoTextsBySentence :: Text -> [[Text]]
  52 monoTextsBySentence = map T.words
  53                     . T.split isSep
  54                     . T.toLower
  55
  56
  57