]> Git — Sourcephile - gargantext.git/blob - src/Gargantext/Text/Terms/Multi/RAKE.hs
[ML] first naive bayes implementation for stop words detection.
[gargantext.git] / src / Gargantext / Text / Terms / Multi / RAKE.hs
1 {-|
2 Module : Gargantext.Text.Terms.Multi.RAKE
3 Description : Rapid automatic keyword extraction (RAKE)
4 Copyright : (c) CNRS, 2017
5 License : AGPL + CECILL v3
6 Maintainer : team@gargantext.org
7 Stability : experimental
8 Portability : POSIX
9
10 Personal notes for the integration of RAKE in Gargantext.
11
12 RAKE algorithm is a simple, rapid and effective algorithm to extract
13 keywords that is very sensitive to the quality of the stop word list.
14
15 Indeed, the very first step starts from the stop words list to cut the
16 text towards keywords extraction. The conTexT is the sentence level to
17 compute the coccurrences and occurrences which are divided to compute
18 the metric of one word. Multi-words metrics is equal to the sum of the
19 metrics of each word.
20
21 Finally The metrics highlight longer keywords which highly depends of
22 quality of the cut which depends on the quality of the stop word list.
23
24 As a consequence, to improve the effectiveness of RAKE algorithm, I am
25 wondering if some bayesian features could be added to increase stop word
26 list quality in time.
27
28 -}
29
30 {-# LANGUAGE NoImplicitPrelude #-}
31
32 module Gargantext.Text.Terms.Multi.RAKE (multiterms_rake, select, hardStopList)
33 where
34
35 import GHC.Real (round)
36 import Data.Text (Text, pack)
37 import NLP.RAKE.Text
38
39 import Gargantext.Text.Terms.Stop (stopList)
40 import Gargantext.Prelude
41
42 select :: Double -> [a] -> [a]
43 select part ns = take n ns
44 where
45 n = round $ part * (fromIntegral $ length ns)
46
47
48 multiterms_rake :: Text -> [WordScore]
49 multiterms_rake = candidates hardStopList
50 defaultNosplit
51 defaultNolist . pSplitter
52
53 -- | StopList
54 hardStopList :: StopwordsMap
55 hardStopList = mkStopwordsStr stopList