src/Gargantext/Ngrams.hs

   1 {-|
   2 Module      : Gargantext.Ngrams
   3 Description : Ngrams tools
   4 Copyright   : (c) CNRS, 2018
   5 License     : AGPL + CECILL v3
   6 Maintainer  : team@gargantext.org
   7 Stability   : experimental
   8 Portability : POSIX
   9
  10 Ngrams exctration.
  11
  12 Definitions of ngrams.
  13 n non negative integer
  14
  15 -}
  16
  17 module Gargantext.Ngrams ( module Gargantext.Ngrams.Letters
  18                               --, module Gargantext.Ngrams.Hetero
  19                          , module Gargantext.Ngrams.CoreNLP
  20                          , module Gargantext.Ngrams.Parser
  21                          , module Gargantext.Ngrams.Occurrences
  22                          , module Gargantext.Ngrams.TextMining
  23                          , module Gargantext.Ngrams.Metrics
  24                          , Ngrams(..), ngrams, occ, sumOcc, text2fis
  25                              --, module Gargantext.Ngrams.Words
  26                          ) where
  27
  28 import Gargantext.Ngrams.Letters
  29 --import Gargantext.Ngrams.Hetero
  30 import Gargantext.Ngrams.CoreNLP
  31 import Gargantext.Ngrams.Parser
  32
  33 import Gargantext.Ngrams.Occurrences
  34 import Gargantext.Ngrams.TextMining
  35 --import Gargantext.Ngrams.Words
  36
  37 import Gargantext.Ngrams.Metrics
  38 import qualified Gargantext.Ngrams.FrequentItemSet as FIS
  39 -----------------------------------------------------------------
  40
  41 import Data.Char (Char, isAlpha, isSpace)
  42 import Data.Text (Text, words, filter, toLower)
  43 import Data.Map.Strict  (Map
  44                         , empty
  45                         , insertWith, unionWith
  46                         , lookupIndex
  47                         --, fromList, keys
  48                         )
  49 import qualified Data.Map.Strict as M (filter)
  50 import Data.Foldable (foldl')
  51 import Gargantext.Prelude hiding (filter)
  52
  53 -- Maybe useful later:
  54 --import NLP.Stemmer (stem, Stemmer(..))
  55 --import Language.Aspell (check, suggest, spellChecker, spellCheckerWithOptions)
  56 --import Language.Aspell.Options (ACOption(..))
  57
  58
  59
  60 data Ngrams = Ngrams { _ngramsNgrams :: Text
  61                      , _ngramsStem   :: Text
  62                      } deriving (Show)
  63
  64 instance Eq Ngrams where
  65   Ngrams n1 s1 == Ngrams n2 s2 = n1 == n2 || s1 == s2
  66
  67
  68 type Occ     = Int
  69 --type Index   = Int
  70
  71 ngrams :: Text -> [Text]
  72 ngrams xs = monograms $ toLower $ filter isChar xs
  73
  74 monograms :: Text -> [Text]
  75 monograms = words
  76
  77 -- TODO
  78 -- 12-b
  79 isChar :: Char -> Bool
  80 isChar '-' = True
  81 isChar '/' = True
  82 isChar  c  = isAlpha c || isSpace c
  83
  84 -- | Compute the occurrences (occ)
  85 occ :: Ord a => [a] -> Map a Occ
  86 occ xs = foldl' (\x y -> insertWith (+) y 1 x) empty xs
  87
  88 -- TODO add groups and filter stops
  89 sumOcc :: Ord a => [Map a Occ] -> Map a Occ
  90 sumOcc xs = foldl' (\x y -> unionWith (+) x y) empty xs
  91
  92 --noApax :: Ord a => Map a Occ -> Map a Occ
  93 --noApax m = M.filter (>1) m
  94
  95 -- | /!\ indexes are not the same:
  96
  97 -- | Index ngrams from Map
  98 --indexNgram :: Ord a => Map a Occ -> Map Index a
  99 --indexNgram m = fromList (zip [1..] (keys m))
 100
 101 -- | Index ngrams from Map
 102 --ngramIndex :: Ord a => Map a Occ -> Map a Index
 103 --ngramIndex m = fromList (zip (keys m) [1..])
 104
 105 indexWith :: Ord a => Map a Occ -> [a] -> [Int]
 106 indexWith m xs = unMaybe $ map (\x -> lookupIndex x m) xs
 107
 108 indexIt :: Ord a => [[a]] -> (Map a Int, [[Int]])
 109 indexIt xs = (m, is)
 110   where
 111     m  = sumOcc (map occ  xs)
 112     is = map    (indexWith m) xs
 113
 114 list2fis :: Ord a => FIS.Frequency -> [[a]] -> (Map a Int, [FIS.Fis])
 115 list2fis n xs = (m', fs)
 116   where
 117     (m, is) = indexIt xs
 118     m'      = M.filter (>50000) m
 119     fs      = FIS.all n is
 120
 121 text2fis :: FIS.Frequency -> [Text] -> (Map Text Int, [FIS.Fis])
 122 text2fis n xs = list2fis n (map ngrams xs)
 123
 124 --text2fisWith :: FIS.Size -> FIS.Frequency -> [Text] -> (Map Text Int, [FIS.Fis])
 125 --text2fisWith = undefined
 126
 127