src/Gargantext/Text.hs

   1 {-|
   2 Module      : Gargantext.Text
   3 Description : Ngrams tools
   4 Copyright   : (c) CNRS, 2018
   5 License     : AGPL + CECILL v3
   6 Maintainer  : team@gargantext.org
   7 Stability   : experimental
   8 Portability : POSIX
   9
  10 Text gathers terms in unit of contexts.
  11
  12 -}
  13
  14 {-# LANGUAGE NoImplicitPrelude #-}
  15 {-# LANGUAGE OverloadedStrings #-}
  16
  17 module Gargantext.Text
  18   where
  19
  20 import qualified Data.Text as DT
  21 --import Data.Text.IO (readFile)
  22
  23
  24 import Data.Text (Text, split)
  25
  26 import NLP.FullStop (segment)
  27 -----------------------------------------------------------------
  28
  29 import Gargantext.Core.Types
  30 import Gargantext.Prelude hiding (filter)
  31 -----------------------------------------------------------------
  32
  33 data Group = Group { _group_label  :: Terms
  34                    , _group_terms :: Terms
  35                    } deriving (Show)
  36
  37
  38 clean :: Text -> Text
  39 clean txt = DT.map clean' txt
  40   where
  41     clean' '’' = '\''
  42     clean' c  = c
  43
  44
  45 --noApax :: Ord a => Map a Occ -> Map a Occ
  46 --noApax m = M.filter (>1) m
  47
  48 -------------------------------------------------------------------
  49 -- Contexts of text
  50 sentences :: Text -> [Text]
  51 sentences txt = map DT.pack $ segment $ DT.unpack txt
  52
  53 sentences' :: Text -> [Text]
  54 sentences' txt = split isStop txt
  55
  56 isStop :: Char -> Bool
  57 isStop c = c `elem` ['.','?','!']
  58
  59 unsentences :: [Text] -> Text
  60 unsentences txts = DT.intercalate " " txts
  61
  62 -- | https://en.wikipedia.org/wiki/Text_mining
  63 testText_en :: Text
  64 testText_en = DT.pack "Text mining, also referred to as text data mining, roughly equivalent to text analytics, is the process of deriving high-quality information from text. High-quality information is typically derived through the devising of patterns and trends through means such as statistical pattern learning. Text mining usually involves the process of structuring the input text (usually parsing, along with the addition of some derived linguistic features and the removal of others, and subsequent insertion into a database), deriving patterns within the structured data, and finally evaluation and interpretation of the output. 'High quality' in text mining usually refers to some combination of relevance, novelty, and interestingness. Typical text mining tasks include text categorization, text clustering, concept/entity extraction, production of granular taxonomies, sentiment analysis, document summarization, and entity relation modeling (i.e., learning relations between named entities). Text analysis involves information retrieval, lexical analysis to study word frequency distributions, pattern recognition, tagging/annotation, information extraction, data mining techniques including link and association analysis, visualization, and predictive analytics. The overarching goal is, essentially, to turn text into data for analysis, via application of natural language processing (NLP) and analytical methods. A typical application is to scan a set of documents written in a natural language and either model the document set for predictive classification purposes or populate a database or search index with the information extracted."
  65
  66 -- | https://fr.wikipedia.org/wiki/Fouille_de_textes
  67 testText_fr :: Text
  68 testText_fr = DT.pack "La fouille de textes ou « l'extraction de connaissances » dans les textes est une spécialisation de la fouille de données et fait partie du domaine de l'intelligence artificielle. Cette technique est souvent désignée sous l'anglicisme text mining. Elle désigne un ensemble de traitements informatiques consistant à extraire des connaissances selon un critère de nouveauté ou de similarité dans des textes produits par des humains pour des humains. Dans la pratique, cela revient à mettre en algorithme un modèle simplifié des théories linguistiques dans des systèmes informatiques d'apprentissage et de statistiques. Les disciplines impliquées sont donc la linguistique calculatoire, l'ingénierie des langues, l'apprentissage artificiel, les statistiques et l'informatique."
  69
  70 -- | Ngrams Test
  71 -- >>> ngramsTest testText
  72 -- 248
  73 --ngramsTest :: Text -> Int
  74 --ngramsTest x =  length ws
  75 --  where
  76 --    --txt = concat <$> lines <$> clean <$> readFile filePath
  77 --    txt = clean x
  78 --    -- | Number of sentences
  79 --    --ls   = sentences $ txt
  80 --    -- | Number of monograms used in the full text
  81 --    ws   = ngrams    $ txt
  82 --    -- | stem ngrams
  83     -- TODO
  84     -- group ngrams
  85     --ocs  = occ       $ ws
  86
  87