]> Git — Sourcephile - gargantext.git/blob - src/Gargantext/Text/Corpus/Parsers/CSV.hs
[refactoring] add some default extensions to package.yaml
[gargantext.git] / src / Gargantext / Text / Corpus / Parsers / CSV.hs
1 {-|
2 Module : Gargantext.Text.Corpus.Parsers.CSV
3 Description :
4 Copyright : (c) CNRS, 2017-Present
5 License : AGPL + CECILL v3
6 Maintainer : team@gargantext.org
7 Stability : experimental
8 Portability : POSIX
9
10 CSV parser for Gargantext corpus files.
11
12 -}
13
14
15 module Gargantext.Text.Corpus.Parsers.CSV where
16
17 import Control.Applicative
18 import Data.Char (ord)
19 import Data.Csv
20 import Data.Either (Either(Left, Right))
21 import Data.Text (Text, pack, length, intercalate)
22 import Data.Time.Segment (jour)
23 import Data.Vector (Vector)
24 import GHC.IO (FilePath)
25 import GHC.Real (round)
26 import GHC.Word (Word8)
27 import Gargantext.Database.Admin.Types.Node -- (HyperdataDocument(..))
28 import Gargantext.Prelude hiding (length)
29 import Gargantext.Text
30 import Gargantext.Text.Context
31 import qualified Data.ByteString as BS
32 import qualified Data.ByteString.Lazy as BL
33 import qualified Data.Vector as V
34
35 ---------------------------------------------------------------
36 headerCsvGargV3 :: Header
37 headerCsvGargV3 = header [ "title"
38 , "source"
39 , "publication_year"
40 , "publication_month"
41 , "publication_day"
42 , "abstract"
43 , "authors"
44 ]
45 ---------------------------------------------------------------
46 data CsvGargV3 = CsvGargV3
47 { d_docId :: !Int
48 , d_title :: !Text
49 , d_source :: !Text
50 , d_publication_year :: !Int
51 , d_publication_month :: !Int
52 , d_publication_day :: !Int
53 , d_abstract :: !Text
54 , d_authors :: !Text
55 }
56 deriving (Show)
57 ---------------------------------------------------------------
58 -- | Doc 2 HyperdataDocument
59 toDoc :: CsvGargV3 -> HyperdataDocument
60 toDoc (CsvGargV3 did dt _ dpy dpm dpd dab dau) =
61 HyperdataDocument (Just "CSV")
62 (Just . pack . show $ did)
63 Nothing
64 Nothing
65 Nothing
66 Nothing
67 (Just dt)
68 Nothing
69 (Just dau)
70 (Just dab)
71 (Nothing)
72 Nothing
73 (Just dpy)
74 (Just dpm)
75 (Just dpd)
76 Nothing
77 Nothing
78 Nothing
79 Nothing
80
81 ---------------------------------------------------------------
82 -- | Types Conversions
83 toDocs :: Vector CsvDoc -> [CsvGargV3]
84 toDocs v = V.toList
85 $ V.zipWith (\nId (CsvDoc t s py pm pd abst auth)
86 -> CsvGargV3 nId t s py pm pd abst auth )
87 (V.enumFromN 1 (V.length v'')) v''
88 where
89 v'' = V.foldl (\v' sep -> V.concatMap (splitDoc (docsSize v') sep) v') v seps
90 seps= (V.fromList [Paragraphs 1, Sentences 3, Chars 3])
91
92 ---------------------------------------------------------------
93 fromDocs :: Vector CsvGargV3 -> Vector CsvDoc
94 fromDocs docs = V.map fromDocs' docs
95 where
96 fromDocs' (CsvGargV3 _ t s py pm pd abst auth) = (CsvDoc t s py pm pd abst auth)
97
98 ---------------------------------------------------------------
99 -- | Split a document in its context
100 -- TODO adapt the size of the paragraph according to the corpus average
101 splitDoc :: Mean -> SplitContext -> CsvDoc -> Vector CsvDoc
102 splitDoc m splt doc = let docSize = (length $ csv_abstract doc) in
103 if docSize > 1000
104 then
105 if (mod (round m) docSize) >= 10
106 then
107 splitDoc' splt doc
108 else
109 V.fromList [doc]
110 else
111 V.fromList [doc]
112 where
113 splitDoc' :: SplitContext -> CsvDoc -> Vector CsvDoc
114 splitDoc' contextSize (CsvDoc t s py pm pd abst auth) = V.fromList $ [firstDoc] <> nextDocs
115 where
116 firstDoc = CsvDoc t s py pm pd firstAbstract auth
117 firstAbstract = head' "splitDoc'1" abstracts
118
119 nextDocs = map (\txt -> CsvDoc
120 (head' "splitDoc'2" $ sentences txt)
121 s py pm pd
122 (unsentences $ tail' "splitDoc'1" $ sentences txt)
123 auth
124 ) (tail' "splitDoc'2" abstracts)
125
126 abstracts = (splitBy $ contextSize) abst
127
128 ---------------------------------------------------------------
129 ---------------------------------------------------------------
130 type Mean = Double
131
132 docsSize :: Vector CsvDoc -> Mean
133 docsSize csvDoc = mean ls
134 where
135 ls = V.toList $ V.map (fromIntegral . length . csv_abstract) csvDoc
136
137
138 ---------------------------------------------------------------
139 data CsvDoc = CsvDoc
140 { csv_title :: !Text
141 , csv_source :: !Text
142 , csv_publication_year :: !Int
143 , csv_publication_month :: !Int
144 , csv_publication_day :: !Int
145 , csv_abstract :: !Text
146 , csv_authors :: !Text
147 }
148 deriving (Show)
149
150 instance FromNamedRecord CsvDoc where
151 parseNamedRecord r = CsvDoc <$> r .: "title"
152 <*> r .: "source"
153 <*> r .: "publication_year"
154 <*> r .: "publication_month"
155 <*> r .: "publication_day"
156 <*> r .: "abstract"
157 <*> r .: "authors"
158
159 instance ToNamedRecord CsvDoc where
160 toNamedRecord (CsvDoc t s py pm pd abst aut) =
161 namedRecord [ "title" .= t
162 , "source" .= s
163 , "publication_year" .= py
164 , "publication_month" .= pm
165 , "publication_day" .= pd
166 , "abstract" .= abst
167 , "authors" .= aut
168 ]
169
170 hyperdataDocument2csvDoc :: HyperdataDocument -> CsvDoc
171 hyperdataDocument2csvDoc h = CsvDoc (m $ _hyperdataDocument_title h)
172 (m $ _hyperdataDocument_source h)
173 (mI $ _hyperdataDocument_publication_year h)
174 (mI $ _hyperdataDocument_publication_month h)
175 (mI $ _hyperdataDocument_publication_day h)
176 (m $ _hyperdataDocument_abstract h)
177 (m $ _hyperdataDocument_authors h)
178
179 where
180 m = maybe "" identity
181 mI = maybe 0 identity
182
183
184 csvDecodeOptions :: DecodeOptions
185 csvDecodeOptions = defaultDecodeOptions {decDelimiter = delimiter}
186
187 csvEncodeOptions :: EncodeOptions
188 csvEncodeOptions = defaultEncodeOptions {encDelimiter = delimiter}
189
190 delimiter :: Word8
191 delimiter = fromIntegral $ ord '\t'
192 ------------------------------------------------------------------------
193 ------------------------------------------------------------------------
194 readCsvOn' :: [CsvDoc -> Text] -> FilePath -> IO [Text]
195 readCsvOn' fields fp = V.toList
196 <$> V.map (\l -> intercalate (pack " ") $ map (\field -> field l) fields)
197 <$> snd
198 <$> readFile fp
199
200 ------------------------------------------------------------------------
201
202 readFileLazy :: (FromNamedRecord a) => proxy a -> FilePath -> IO (Header, Vector a)
203 readFileLazy f = fmap (readByteStringLazy f) . BL.readFile
204
205 readFileStrict :: (FromNamedRecord a) => proxy a -> FilePath -> IO (Header, Vector a)
206 readFileStrict f = fmap (readByteStringStrict f) . BS.readFile
207
208 readByteStringLazy :: (FromNamedRecord a) => proxy a -> BL.ByteString -> (Header, Vector a)
209 readByteStringLazy _f bs = case decodeByNameWith csvDecodeOptions bs of
210 Left e -> panic (pack e)
211 Right csvDocs -> csvDocs
212
213 readByteStringStrict :: (FromNamedRecord a) => proxy a -> BS.ByteString -> (Header, Vector a)
214 readByteStringStrict ff = (readByteStringLazy ff) . BL.fromStrict
215
216 ------------------------------------------------------------------------
217 -- | TODO use readFileLazy
218 readFile :: FilePath -> IO (Header, Vector CsvDoc)
219 readFile = fmap readCsvLazyBS . BL.readFile
220
221
222 -- | TODO use readByteStringLazy
223 readCsvLazyBS :: BL.ByteString -> (Header, Vector CsvDoc)
224 readCsvLazyBS bs = case decodeByNameWith csvDecodeOptions bs of
225 Left e -> panic (pack e)
226 Right csvDocs -> csvDocs
227
228 ------------------------------------------------------------------------
229
230 -- | TODO use readFileLazy
231 readCsvHal :: FilePath -> IO (Header, Vector CsvHal)
232 readCsvHal = fmap readCsvHalLazyBS . BL.readFile
233
234 -- | TODO use readByteStringLazy
235 readCsvHalLazyBS :: BL.ByteString -> (Header, Vector CsvHal)
236 readCsvHalLazyBS bs = case decodeByNameWith csvDecodeOptions bs of
237 Left e -> panic (pack e)
238 Right csvDocs -> csvDocs
239
240 readCsvHalBSStrict :: BS.ByteString -> (Header, Vector CsvHal)
241 readCsvHalBSStrict = readCsvHalLazyBS . BL.fromStrict
242
243 ------------------------------------------------------------------------
244 writeFile :: FilePath -> (Header, Vector CsvDoc) -> IO ()
245 writeFile fp (h, vs) = BL.writeFile fp $
246 encodeByNameWith csvEncodeOptions h (V.toList vs)
247
248 writeDocs2Csv :: FilePath -> [HyperdataDocument] -> IO ()
249 writeDocs2Csv fp hs = BL.writeFile fp $ hyperdataDocument2csv hs
250
251 hyperdataDocument2csv :: [HyperdataDocument] -> BL.ByteString
252 hyperdataDocument2csv hs = encodeByNameWith csvEncodeOptions headerCsvGargV3 (map hyperdataDocument2csvDoc hs)
253
254 ------------------------------------------------------------------------
255 -- Hal Format
256 data CsvHal = CsvHal
257 { csvHal_title :: !Text
258 , csvHal_source :: !Text
259 , csvHal_publication_year :: !Integer
260 , csvHal_publication_month :: !Int
261 , csvHal_publication_day :: !Int
262 , csvHal_abstract :: !Text
263 , csvHal_authors :: !Text
264
265 , csvHal_url :: !Text
266 , csvHal_isbn_s :: !Text
267 , csvHal_issue_s :: !Text
268 , csvHal_journalPublisher_s:: !Text
269 , csvHal_language_s :: !Text
270
271 , csvHal_doiId_s :: !Text
272 , csvHal_authId_i :: !Text
273 , csvHal_instStructId_i :: !Text
274 , csvHal_deptStructId_i :: !Text
275 , csvHal_labStructId_i :: !Text
276
277 , csvHal_rteamStructId_i :: !Text
278 , csvHal_docType_s :: !Text
279 }
280 deriving (Show)
281
282 instance FromNamedRecord CsvHal where
283 parseNamedRecord r = CsvHal <$> r .: "title"
284 <*> r .: "source"
285 <*> r .: "publication_year"
286 <*> r .: "publication_month"
287 <*> r .: "publication_day"
288 <*> r .: "abstract"
289 <*> r .: "authors"
290
291 <*> r .: "url"
292 <*> r .: "isbn_s"
293 <*> r .: "issue_s"
294 <*> r .: "journalPublisher_s"
295 <*> r .: "language_s"
296
297 <*> r .: "doiId_s"
298 <*> r .: "authId_i"
299 <*> r .: "instStructId_i"
300 <*> r .: "deptStructId_i"
301 <*> r .: "labStructId_i"
302
303 <*> r .: "rteamStructId_i"
304 <*> r .: "docType_s"
305
306 instance ToNamedRecord CsvHal where
307 toNamedRecord (CsvHal t s py pm pd abst aut url isbn iss j lang doi auth inst dept lab team doct) =
308 namedRecord [ "title" .= t
309 , "source" .= s
310
311 , "publication_year" .= py
312 , "publication_month" .= pm
313 , "publication_day" .= pd
314
315 , "abstract" .= abst
316 , "authors" .= aut
317
318 , "url" .= url
319 , "isbn_s" .= isbn
320 , "issue_s" .= iss
321 , "journalPublisher_s" .= j
322 , "language_s" .= lang
323
324 , "doiId_s" .= doi
325 , "authId_i" .= auth
326 , "instStructId_i" .= inst
327 , "deptStructId_i" .= dept
328 , "labStructId_i" .= lab
329
330 , "rteamStructId_i" .= team
331 , "docType_s" .= doct
332 ]
333
334 csvHal2doc :: CsvHal -> HyperdataDocument
335 csvHal2doc (CsvHal title source
336 pub_year pub_month pub_day
337 abstract authors
338 url _ _ _ _
339 doi _ inst _ _
340 _ _ ) = HyperdataDocument (Just "CsvHal")
341 (Just doi)
342 (Just url)
343 Nothing
344 Nothing
345 Nothing
346 (Just title)
347 (Just authors)
348 (Just inst)
349 (Just source)
350 (Just abstract)
351 (Just $ pack . show $ jour pub_year pub_month pub_day)
352 (Just $ fromIntegral pub_year)
353 (Just pub_month)
354 (Just pub_day)
355 Nothing
356 Nothing
357 Nothing
358 Nothing
359
360
361 csv2doc :: CsvDoc -> HyperdataDocument
362 csv2doc (CsvDoc title source
363 pub_year pub_month pub_day
364 abstract authors ) = HyperdataDocument (Just "CsvHal")
365 Nothing
366 Nothing
367 Nothing
368 Nothing
369 Nothing
370 (Just title)
371 (Just authors)
372 Nothing
373 (Just source)
374 (Just abstract)
375 (Just $ pack . show $ jour (fromIntegral pub_year) pub_month pub_day)
376 (Just $ fromIntegral pub_year)
377 (Just pub_month)
378 (Just pub_day)
379 Nothing
380 Nothing
381 Nothing
382 Nothing
383
384 ------------------------------------------------------------------------
385 parseHal :: FilePath -> IO [HyperdataDocument]
386 parseHal fp = V.toList <$> V.map csvHal2doc <$> snd <$> readCsvHal fp
387
388 parseHal' :: BL.ByteString -> [HyperdataDocument]
389 parseHal' = V.toList . V.map csvHal2doc . snd . readCsvHalLazyBS
390
391 ------------------------------------------------------------------------
392
393 parseCsv :: FilePath -> IO [HyperdataDocument]
394 parseCsv fp = V.toList <$> V.map csv2doc <$> snd <$> readFile fp
395
396 parseCsv' :: BL.ByteString -> [HyperdataDocument]
397 parseCsv' bs = V.toList $ V.map csv2doc $ snd $ readCsvLazyBS bs
398