]> Git — Sourcephile - gargantext.git/blob - src/Gargantext/Text/Metrics/FrequentItemSet.hs
Merge branch 'dev' into dev-doc-table-optimization
[gargantext.git] / src / Gargantext / Text / Metrics / FrequentItemSet.hs
1 {-|
2 Module : Gargantext.Text.Metrics.FrequentItemSet
3 Description : Ngrams tools
4 Copyright : (c) CNRS, 2018
5 License : AGPL + CECILL v3
6 Maintainer : team@gargantext.org
7 Stability : experimental
8 Portability : POSIX
9
10 Domain Specific Language to manage Frequent Item Set (FIS)
11
12 -}
13
14
15 module Gargantext.Text.Metrics.FrequentItemSet
16 ( Fis, Size(..)
17 , occ_hlcm, cooc_hlcm
18 , allFis, between
19 , fisWithSize
20 , fisWith
21 , fisWithSizePoly
22 , fisWithSizePoly2
23 , fisWithSizePolyMap
24 , module HLCM
25 )
26 where
27
28 import Prelude (Functor(..)) -- TODO
29 import qualified Data.Map.Strict as Map
30 import Data.Map.Strict (Map)
31 import qualified Data.Set as Set
32 import Data.Set (Set)
33 import qualified Data.Vector as V
34
35 import Data.List (filter, concat, null)
36 import Data.Maybe (catMaybes)
37
38 import HLCM
39
40 import Gargantext.Prelude
41
42 data Size = Point Int | Segment Int Int
43
44 ------------------------------------------------------------------------
45 -- | Occurrence is Frequent Item Set of size 1
46 occ_hlcm :: Frequency -> [[Item]] -> [Fis]
47 occ_hlcm = fisWithSize (Point 1)
48
49 -- | Cooccurrence is Frequent Item Set of size 2
50 cooc_hlcm :: Frequency -> [[Item]] -> [Fis]
51 cooc_hlcm = fisWithSize (Point 2)
52
53 allFis :: Frequency -> [[Item]] -> [Fis]
54 allFis = fisWith Nothing
55
56 ------------------------------------------------------------------------
57 between :: (Int, Int) -> Frequency -> [[Item]] -> [Fis]
58 between (x,y) = fisWithSize (Segment x y)
59
60 --maximum :: Int -> Frequency -> [[Item]] -> [Fis]
61 --maximum m = between (0,m)
62
63
64 ------------------------------------------------------------------------
65 ------------------------------------------------------------------------
66 -- | Data type to type the Frequent Item Set
67 -- TODO replace List with Set in fisItemSet
68 -- be careful : risks to erase HLCM behavior
69 type Fis = Fis' Item
70 data Fis' a = Fis' { _fisCount :: Int
71 , _fisItemSet :: [a]
72 } deriving (Show)
73
74 instance Functor Fis' where
75 fmap f (Fis' c is) = Fis' c (fmap f is)
76
77 -- | Sugar from items to FIS
78 items2fis :: [Item] -> Maybe Fis
79 items2fis [] = Nothing
80 items2fis (i:is) = Just $ Fis' i is
81
82 ------------------------------------------------------------------------
83 ------------------------------------------------------------------------
84
85 fisWithSize :: Size -> Frequency -> [[Item]] -> [Fis]
86 fisWithSize n f is = case n of
87 Point n' -> fisWith (Just (\x -> length x == (n'+1) )) f is
88 Segment a b -> fisWith (Just (\x -> cond a (length x) b)) f is
89 where
90 cond a' x b' = a' <= x && x <= b'
91
92
93 --- Filter on Fis and not on [Item]
94 fisWith :: Maybe ([Item] -> Bool) -> Frequency -> [[Item]] -> [Fis]
95 fisWith s f is = case filter (not . null) is of
96 [] -> []
97 js -> catMaybes $ map items2fis $ filter' $ runLCMmatrix js f
98 -- drop unMaybe
99 where
100 filter' = case s of
101 Nothing -> identity
102 Just fun -> filter fun
103
104 -- Here the sole purpose to take the keys as a Set is tell we do not want
105 -- duplicates.
106 fisWithSizePoly :: Ord a => Size -> Frequency -> Set a -> [[a]] -> [Fis' a]
107 fisWithSizePoly n f ks = map (fmap fromItem) . fisWithSize n f . map (map toItem)
108 where
109 ksv = V.fromList $ Set.toList ks
110 ksm = Map.fromList . flip zip [0..] $ V.toList ksv
111 toItem = (ksm Map.!)
112 fromItem = (ksv V.!)
113
114 fisWithSizePoly2 :: Ord a => Size -> Frequency -> [[a]] -> [Fis' a]
115 fisWithSizePoly2 n f is = fisWithSizePoly n f ks is
116 where
117 ks = Set.fromList $ concat is
118
119 fisWithSizePolyMap :: Ord a => Size -> Frequency -> [[a]] -> Map (Set a) Int
120 fisWithSizePolyMap n f is =
121 Map.fromList $ (\i -> (Set.fromList (_fisItemSet i), _fisCount i)) <$> fisWithSizePoly2 n f is
122
123
124 ------------------------------------------------------------------------
125 ------------------------------------------------------------------------
126
127
128 --
129 ---- | /!\ indexes are not the same:
130 --
131 ---- | Index ngrams from Map
132 ----indexNgram :: Ord a => Map a Occ -> Map Index a
133 ----indexNgram m = fromList (zip [1..] (keys m))
134 --
135 ---- | Index ngrams from Map
136 ----ngramIndex :: Ord a => Map a Occ -> Map a Index
137 ----ngramIndex m = fromList (zip (keys m) [1..])
138 --
139 --indexWith :: Ord a => Map a Occ -> [a] -> [Int]
140 --indexWith m xs = unMaybe $ map (\x -> lookupIndex x m) xs
141 --
142 --indexIt :: Ord a => [[a]] -> (Map a Int, [[Int]])
143 --indexIt xs = (m, is)
144 -- where
145 -- m = sumOcc (map occ xs)
146 -- is = map (indexWith m) xs
147 --
148 --list2fis :: Ord a => FIS.Frequency -> [[a]] -> (Map a Int, [FIS.Fis])
149 --list2fis n xs = (m', fs)
150 -- where
151 -- (m, is) = indexIt xs
152 -- m' = M.filter (>50000) m
153 -- fs = FIS.all n is
154 --
155 --text2fis :: FIS.Frequency -> [Text] -> (Map Text Int, [FIS.Fis])
156 --text2fis n xs = list2fis n (map terms xs)
157 --
158 ----text2fisWith :: FIS.Size -> FIS.Frequency -> [Text] -> (Map Text Int, [FIS.Fis])
159 ----text2fisWith = undefined
160 --
161