replaced HTML web scraping examples

Author: mark-watson

WebScraping/HandsomeSoupTest.hs

@@ -3,14 +3,13 @@
 ## Run examples using stack
 
 ````````
-stack build --exec HttpClientExample
-stack build --exec HandsomeSoupTest
+stack build --exec TagSoupTest
 ````````
 
 ## Run examples using cabal:
 
 ````````
 cabal build
-cabal run HandsomeSoupTest
+cabal run TagSoupTest
 ````````
 

WebScraping/HttpClientExample.hs

@@ -0,0 +1,42 @@
+{-# LANGUAGE OverloadedStrings #-}
+
+import Network.HTTP.Simple
+import Text.HTML.TagSoup
+import Data.Text (Text)
+import qualified Data.Text as T
+import qualified Data.Text.IO as TIO
+import qualified Data.ByteString.Lazy.Char8 as BL8
+import Data.Maybe (mapMaybe)
+
+main :: IO ()
+main = do
+    -- Fetch the HTML content
+    response <- httpLBS "https://markwatson.com/"
+    let body = BL8.unpack $ getResponseBody response
+        tags = parseTags body
+
+    -- Extract and print headers
+    let headers = getResponseHeaders response
+    putStrLn "Headers:"
+    mapM_ print headers
+
+    -- Extract and print all text content
+    let texts = extractTexts tags
+    putStrLn "\nText Content:"
+    TIO.putStrLn texts
+
+    -- Extract and print all links
+    let links = extractLinks tags
+    putStrLn "\nLinks:"
+    mapM_ TIO.putStrLn links
+
+-- Function to extract all text content from tags
+extractTexts :: [Tag String] -> Text
+extractTexts = T.unwords . map (T.strip . T.pack) . filter (not . null) . mapMaybe maybeTagText
+
+-- Function to extract all links from tags
+extractLinks :: [Tag String] -> [Text]
+extractLinks = map (T.pack . fromAttrib "href") . filter isATag
+  where
+    isATag (TagOpen "a" _) = True
+    isATag _               = False

WebScraping/README.md

@@ -12,16 +12,10 @@ category:            dev
 build-type:          Simple
 cabal-version:       >=1.22.4.0
 
-executable HttpClientExample
+executable TagSoupTest
   hs-source-dirs:      .
-  main-is:             HttpClientExample.hs
+  main-is:             TagSoupTest.hs
   default-language:    Haskell2010
   build-depends:       base >= 4.7 && < 5, wreq, lens, bytestring,
-                       hxt, json >= 0.10
-
-executable HandsomeSoupTest
-  hs-source-dirs:      .
-  main-is:             HandsomeSoupTest.hs
-  default-language:    Haskell2010
-  build-depends:       base >= 4.7 && < 5, wreq, lens, bytestring,
-                       HandsomeSoup, hxt, json >= 0.10
+                       json >= 0.10, text >=2.1, 
+                       http-conduit >= 2.3.8.3, tagsoup >= 0.14.8