src/Gargantext/Text/Metrics/FrequentItemSet.hs

   1 {-|
   2 Module      : Gargantext.Text.Metrics.FrequentItemSet
   3 Description : Ngrams tools
   4 Copyright   : (c) CNRS, 2018
   5 License     : AGPL + CECILL v3
   6 Maintainer  : team@gargantext.org
   7 Stability   : experimental
   8 Portability : POSIX
   9
  10 Domain Specific Language to manage Frequent Item Set (FIS)
  11
  12 -}
  13
  14 {-# LANGUAGE NoImplicitPrelude #-}
  15
  16 module Gargantext.Text.Metrics.FrequentItemSet
  17   ( Fis, Size(..)
  18   , occ_hlcm, cooc_hlcm
  19   , allFis, between
  20   , fisWithSize
  21   , fisWith
  22   , fisWithSizePoly
  23   , fisWithSizePoly2
  24   , fisWithSizePolyMap
  25   , module HLCM
  26   )
  27   where
  28
  29 import Prelude (Functor(..)) -- TODO
  30 import qualified Data.Map.Strict as Map
  31 import Data.Map.Strict (Map)
  32 import qualified Data.Set as Set
  33 import Data.Set (Set)
  34 import qualified Data.Vector as V
  35
  36 import Data.List (filter, concat, null)
  37 import Data.Maybe (catMaybes)
  38
  39 import HLCM
  40
  41 import Gargantext.Prelude
  42
  43 data Size = Point Int | Segment Int Int
  44
  45 ------------------------------------------------------------------------
  46 -- | Occurrence is Frequent Item Set of size 1
  47 occ_hlcm  :: Frequency -> [[Item]] -> [Fis]
  48 occ_hlcm = fisWithSize (Point 1)
  49
  50 -- | Cooccurrence is Frequent Item Set of size 2
  51 cooc_hlcm :: Frequency -> [[Item]] -> [Fis]
  52 cooc_hlcm = fisWithSize (Point 2)
  53
  54 allFis :: Frequency -> [[Item]] -> [Fis]
  55 allFis = fisWith Nothing
  56
  57 ------------------------------------------------------------------------
  58 between :: (Int, Int) -> Frequency -> [[Item]] -> [Fis]
  59 between (x,y) = fisWithSize (Segment x y)
  60
  61 --maximum :: Int -> Frequency -> [[Item]] -> [Fis]
  62 --maximum m = between (0,m)
  63
  64
  65 ------------------------------------------------------------------------
  66 ------------------------------------------------------------------------
  67 -- | Data type to type the Frequent Item Set
  68 -- TODO replace List with Set in fisItemSet
  69 -- be careful : risks to erase HLCM behavior
  70 type Fis = Fis' Item
  71 data Fis' a = Fis' { _fisCount   :: Int
  72                    , _fisItemSet :: [a]
  73                    } deriving (Show)
  74
  75 instance Functor Fis' where
  76   fmap f (Fis' c is) = Fis' c (fmap f is)
  77
  78 -- | Sugar from items to FIS
  79 items2fis :: [Item] -> Maybe Fis
  80 items2fis []     = Nothing
  81 items2fis (i:is) = Just $ Fis' i is
  82
  83 ------------------------------------------------------------------------
  84 ------------------------------------------------------------------------
  85
  86 fisWithSize :: Size -> Frequency -> [[Item]] -> [Fis]
  87 fisWithSize n f is = case n of
  88   Point  n'   -> fisWith (Just (\x -> length x == (n'+1) )) f is
  89   Segment a b -> fisWith (Just (\x -> cond a (length x) b)) f is
  90                     where
  91                       cond a' x b' = a' <= x && x <= b'
  92
  93
  94                       --- Filter on Fis and not on [Item]
  95 fisWith :: Maybe ([Item] -> Bool) -> Frequency -> [[Item]] -> [Fis]
  96 fisWith s f is = case filter (not . null) is of
  97                    [] -> []
  98                    js -> catMaybes $ map items2fis $ filter' $ runLCMmatrix js f
  99 -- drop unMaybe
 100   where
 101     filter' = case s of
 102                 Nothing  -> identity
 103                 Just fun -> filter fun
 104
 105 -- Here the sole purpose to take the keys as a Set is tell we do not want
 106 -- duplicates.
 107 fisWithSizePoly :: Ord a => Size -> Frequency -> Set a -> [[a]] -> [Fis' a]
 108 fisWithSizePoly n f ks = map (fmap fromItem) . fisWithSize n f . map (map toItem)
 109   where
 110     ksv = V.fromList $ Set.toList ks
 111     ksm = Map.fromList . flip zip [0..] $ V.toList ksv
 112     toItem = (ksm Map.!)
 113     fromItem = (ksv V.!)
 114
 115 fisWithSizePoly2 :: Ord a => Size -> Frequency -> [[a]] -> [Fis' a]
 116 fisWithSizePoly2 n f is = fisWithSizePoly n f ks is
 117   where
 118     ks = Set.fromList $ concat is
 119
 120 fisWithSizePolyMap :: Ord a => Size -> Frequency -> [[a]] -> Map (Set a) Int
 121 fisWithSizePolyMap n f is =
 122   Map.fromList $ (\i -> (Set.fromList (_fisItemSet i), _fisCount i)) <$> fisWithSizePoly2 n f is
 123
 124
 125 ------------------------------------------------------------------------
 126 ------------------------------------------------------------------------
 127
 128
 129 --
 130 ---- | /!\ indexes are not the same:
 131 --
 132 ---- | Index ngrams from Map
 133 ----indexNgram :: Ord a => Map a Occ -> Map Index a
 134 ----indexNgram m = fromList (zip [1..] (keys m))
 135 --
 136 ---- | Index ngrams from Map
 137 ----ngramIndex :: Ord a => Map a Occ -> Map a Index
 138 ----ngramIndex m = fromList (zip (keys m) [1..])
 139 --
 140 --indexWith :: Ord a => Map a Occ -> [a] -> [Int]
 141 --indexWith m xs = unMaybe $ map (\x -> lookupIndex x m) xs
 142 --
 143 --indexIt :: Ord a => [[a]] -> (Map a Int, [[Int]])
 144 --indexIt xs = (m, is)
 145 --  where
 146 --    m  = sumOcc (map occ  xs)
 147 --    is = map    (indexWith m) xs
 148 --
 149 --list2fis :: Ord a => FIS.Frequency -> [[a]] -> (Map a Int, [FIS.Fis])
 150 --list2fis n xs = (m', fs)
 151 --  where
 152 --    (m, is) = indexIt xs
 153 --    m'      = M.filter (>50000) m
 154 --    fs      = FIS.all n is
 155 --
 156 --text2fis :: FIS.Frequency -> [Text] -> (Map Text Int, [FIS.Fis])
 157 --text2fis n xs = list2fis n (map terms xs)
 158 --
 159 ----text2fisWith :: FIS.Size -> FIS.Frequency -> [Text] -> (Map Text Int, [FIS.Fis])
 160 ----text2fisWith = undefined
 161 --
 162