src/Data/Gargantext/Ngrams/Count.hs

   1 {-# LANGUAGE OverloadedStrings #-}
   2
   3 module Data.Gargantext.Ngrams.Count where
   4
   5 import System.Environment (getArgs)
   6
   7 import Data.List (foldl', take)
   8 import Data.Foldable as F
   9
  10 import Data.Map (Map)
  11 import qualified Data.Map as M
  12
  13 import qualified Data.Text.Lazy.IO as DTLIO
  14 import qualified Data.Text.Lazy as DTL
  15
  16 -- | /O(n)/ Breaks a 'Text' up into each Text list of chars.
  17 -- from slower to faster:
  18 letters :: DTL.Text -> [DTL.Text]
  19 letters text = DTL.chunksOf 1 text
  20
  21 letters' :: DTL.Text -> [DTL.Text]
  22 letters' text = DTL.splitOn "#" $ DTL.intersperse '#' text
  23
  24 letters'' :: DTL.Text -> [DTL.Text]
  25 letters'' = DTL.foldr (\ch xs -> DTL.singleton ch : xs) []
  26
  27
  28 -- words
  29 -- lines
  30 -- words between punctuation
  31 -- number of punctuation
  32
  33 occurrences :: Ord a => [a] -> Map a Int
  34 occurrences xs = foldl' (\x y -> M.insertWith' (+) y 1 x) M.empty xs
  35
  36 -- for optimization :
  37 --occurrences' :: Ord a => [a] -> Map a Integer
  38 --occurrences' xs = DTL.foldl (\x y -> M.insertWith' (+) y 1 x) M.empty xs
  39
  40
  41 countMain = do
  42   (fichier:rest) <- getArgs
  43   c <- DTLIO.readFile fichier
  44   --print $ occurrences $ DTL.chunksOf 1 c
  45   print $ occurrences $ letters'' c
  46   --print $ occurrences $ DTL.words $ DTL.toLower c
  47