src/Gargantext/Text.hs

   1 {-|
   2 Module      : Gargantext.Text
   3 Description : Ngrams tools
   4 Copyright   : (c) CNRS, 2018
   5 License     : AGPL + CECILL v3
   6 Maintainer  : team@gargantext.org
   7 Stability   : experimental
   8 Portability : POSIX
   9
  10 Ngrams exctration.
  11
  12 Definitions of ngrams.
  13 n non negative integer
  14
  15 -}
  16
  17 {-# LANGUAGE NoImplicitPrelude #-}
  18 {-# LANGUAGE OverloadedStrings #-}
  19
  20 module Gargantext.Text
  21   where
  22
  23 import qualified Data.Text as DT
  24 --import Data.Text.IO (readFile)
  25
  26
  27 import Data.Map.Strict  (Map
  28                         , lookupIndex
  29                         --, fromList, keys
  30                         )
  31
  32 import Data.Text (Text, split)
  33 import qualified Data.Map.Strict as M (filter)
  34
  35 -----------------------------------------------------------------
  36 import Gargantext.Text.Ngrams
  37 import Gargantext.Text.Metrics.Occurrences
  38
  39 import qualified Gargantext.Text.Metrics.FrequentItemSet as FIS
  40 import Gargantext.Prelude hiding (filter)
  41 -----------------------------------------------------------------
  42
  43 data Group = Group { _group_label  ::  Ngrams
  44                    , _group_ngrams :: [Ngrams]
  45                    } deriving (Show)
  46
  47
  48 clean :: Text -> Text
  49 clean txt = DT.map clean' txt
  50   where
  51     clean' '’' = '\''
  52     clean' c  = c
  53
  54 --noApax :: Ord a => Map a Occ -> Map a Occ
  55 --noApax m = M.filter (>1) m
  56
  57 -- | /!\ indexes are not the same:
  58
  59 -- | Index ngrams from Map
  60 --indexNgram :: Ord a => Map a Occ -> Map Index a
  61 --indexNgram m = fromList (zip [1..] (keys m))
  62
  63 -- | Index ngrams from Map
  64 --ngramIndex :: Ord a => Map a Occ -> Map a Index
  65 --ngramIndex m = fromList (zip (keys m) [1..])
  66
  67 indexWith :: Ord a => Map a Occ -> [a] -> [Int]
  68 indexWith m xs = unMaybe $ map (\x -> lookupIndex x m) xs
  69
  70 indexIt :: Ord a => [[a]] -> (Map a Int, [[Int]])
  71 indexIt xs = (m, is)
  72   where
  73     m  = sumOcc (map occ  xs)
  74     is = map    (indexWith m) xs
  75
  76 list2fis :: Ord a => FIS.Frequency -> [[a]] -> (Map a Int, [FIS.Fis])
  77 list2fis n xs = (m', fs)
  78   where
  79     (m, is) = indexIt xs
  80     m'      = M.filter (>50000) m
  81     fs      = FIS.all n is
  82
  83 text2fis :: FIS.Frequency -> [Text] -> (Map Text Int, [FIS.Fis])
  84 text2fis n xs = list2fis n (map ngrams xs)
  85
  86 --text2fisWith :: FIS.Size -> FIS.Frequency -> [Text] -> (Map Text Int, [FIS.Fis])
  87 --text2fisWith = undefined
  88
  89 -------------------------------------------------------------------
  90 -- Contexts of text
  91 sentences :: Text -> [Text]
  92 sentences txt = split isStop txt
  93
  94 isStop :: Char -> Bool
  95 isStop c = c `elem` ['.','?','!']
  96
  97
  98 -- | https://en.wikipedia.org/wiki/Text_mining
  99 testText_en :: Text
 100 testText_en = DT.pack "Text mining, also referred to as text data mining, roughly equivalent to text analytics, is the process of deriving high-quality information from text. High-quality information is typically derived through the devising of patterns and trends through means such as statistical pattern learning. Text mining usually involves the process of structuring the input text (usually parsing, along with the addition of some derived linguistic features and the removal of others, and subsequent insertion into a database), deriving patterns within the structured data, and finally evaluation and interpretation of the output. 'High quality' in text mining usually refers to some combination of relevance, novelty, and interestingness. Typical text mining tasks include text categorization, text clustering, concept/entity extraction, production of granular taxonomies, sentiment analysis, document summarization, and entity relation modeling (i.e., learning relations between named entities). Text analysis involves information retrieval, lexical analysis to study word frequency distributions, pattern recognition, tagging/annotation, information extraction, data mining techniques including link and association analysis, visualization, and predictive analytics. The overarching goal is, essentially, to turn text into data for analysis, via application of natural language processing (NLP) and analytical methods. A typical application is to scan a set of documents written in a natural language and either model the document set for predictive classification purposes or populate a database or search index with the information extracted."
 101
 102 -- | https://fr.wikipedia.org/wiki/Fouille_de_textes
 103 testText_fr :: Text
 104 testText_fr = DT.pack "La fouille de textes ou « l'extraction de connaissances » dans les textes est une spécialisation de la fouille de données et fait partie du domaine de l'intelligence artificielle. Cette technique est souvent désignée sous l'anglicisme text mining. Elle désigne un ensemble de traitements informatiques consistant à extraire des connaissances selon un critère de nouveauté ou de similarité dans des textes produits par des humains pour des humains. Dans la pratique, cela revient à mettre en algorithme un modèle simplifié des théories linguistiques dans des systèmes informatiques d'apprentissage et de statistiques. Les disciplines impliquées sont donc la linguistique calculatoire, l'ingénierie des langues, l'apprentissage artificiel, les statistiques et l'informatique."
 105
 106 -- | Ngrams Test
 107 -- >>> ngramsTest testText
 108 -- 248
 109 ngramsTest :: Text -> Int
 110 ngramsTest x=  length ws
 111   where
 112     --txt = concat <$> lines <$> clean <$> readFile filePath
 113     txt = clean x
 114     -- | Number of sentences
 115     --ls   = sentences $ txt
 116     -- | Number of monograms used in the full text
 117     ws   = ngrams    $ txt
 118     -- | stem ngrams
 119     -- TODO
 120     -- group ngrams
 121     --ocs  = occ       $ ws
 122