src/Gargantext/Text/Terms.hs

   1 {-|
   2 Module      : Gargantext.Text.Ngrams
   3 Description : Ngrams definition and tools
   4 Copyright   : (c) CNRS, 2017 - present
   5 License     : AGPL + CECILL v3
   6 Maintainer  : team@gargantext.org
   7 Stability   : experimental
   8 Portability : POSIX
   9
  10 An @n-gram@ is a contiguous sequence of n items from a given sample of
  11 text. In Gargantext application the items are words, n is a non negative
  12 integer.
  13
  14 Using Latin numerical prefixes, an n-gram of size 1 is referred to as a
  15 "unigram"; size 2 is a "bigram" (or, less commonly, a "digram"); size
  16 3 is a "trigram". English cardinal numbers are sometimes used, e.g.,
  17 "four-gram", "five-gram", and so on.
  18
  19 Source: https://en.wikipedia.org/wiki/Ngrams
  20
  21 TODO
  22 group Ngrams -> Tree
  23 compute occ by node of Tree
  24 group occs according groups
  25
  26 compute cooccurrences
  27 compute graph
  28
  29 -}
  30
  31 {-# LANGUAGE NoImplicitPrelude #-}
  32
  33 module Gargantext.Text.Terms
  34   where
  35
  36 import Data.List (concat)
  37 import Data.Text (Text)
  38 import Data.Traversable
  39
  40 import Gargantext.Prelude
  41 import Gargantext.Core
  42 import Gargantext.Core.Types
  43 import Gargantext.Text.Terms.Multi (multiterms)
  44 import Gargantext.Text.Terms.Mono  (monoTerms)
  45 import Gargantext.Text.Terms.WithList (Patterns, extractTermsWithList)
  46
  47
  48 data TermType lang = Mono lang | Multi lang | MonoMulti lang | WithList Patterns
  49
  50
  51 group :: [Text] -> [Text]
  52 group = undefined
  53
  54 -- remove Stop Words
  55 -- map (filter (\t -> not . elem t)) $
  56 ------------------------------------------------------------------------
  57 -- | Sugar to extract terms from text (hiddeng mapM from end user).
  58 --extractTerms :: Traversable t => TermType Lang -> t Text -> IO (t [Terms])
  59 extractTerms :: TermType Lang -> [Text] -> IO [[Terms]]
  60 extractTerms termTypeLang = mapM (terms termTypeLang)
  61 ------------------------------------------------------------------------
  62 -- | Terms from Text
  63 -- Mono : mono terms
  64 -- Multi : multi terms
  65 -- MonoMulti : mono and multi
  66 -- TODO : multi terms should exclude mono (intersection is not empty yet)
  67 terms :: TermType Lang -> Text -> IO [Terms]
  68 terms (Mono      lang) txt = pure $ monoTerms lang txt
  69 terms (Multi     lang) txt = multiterms lang txt
  70 terms (MonoMulti lang) txt = terms (Multi lang) txt
  71 terms (WithList  list) txt = pure . concat $ extractTermsWithList list txt
  72 ------------------------------------------------------------------------
  73