redditlist.com has changed frontend

get_subreddits/download.sh

@@ -1,11 +1,11 @@
 rm -r sfw
 mkdir sfw
-for i in $(seq 1 169); do
-    wget -O ./sfw/$i.txt http://redditlist.com/page-$i
+for i in $(seq 1 34); do
+    wget -O ./sfw/$i.txt http://redditlist.com/?page=$i
 done
 
 rm -r nsfw
 mkdir nsfw
-for i in $(seq 1 29); do
-    wget -O ./nsfw/$i.txt http://redditlist.com/nsfw/page-$i
+for i in $(seq 1 7); do
+    wget -O ./nsfw/$i.txt http://redditlist.com/nsfw?page=$i
 done

get_subreddits/parse.sh

@@ -1,15 +1,17 @@
 function parse {
-    grep -r $1 -e '.*' | grep -e "reddit.com\/r\/" | grep -v "%" | perl -pe 's/.*reddit.com.r.[^\/]*\/"\>([^\<]*).............([0-9]*).*/ , "\1,\2"/g' > temp_list.txt
+    grep -r $1 -e '.*' | grep -B 2 listing-stat | perl -pe "s/\n/newline/g" | perl -pe "s/reddit.com.r./\n/g" | perl -pe "s/(.*). target.*listing-stat..(.*)..span.*/\1;\2/g" | perl -pe "s/,//g" | perl -pe "s/;/,/g" | grep -v ".txt:" | perl -pe 's/^(.*)/  , "\1"/g' > temp_list.txt
 
     echo module $2 where > $2.elm.temp
     echo "$1Raw : List String" >> $2.elm.temp
     echo $1Raw = [ >> $2.elm.temp
     cat temp_list.txt | sort | uniq >> $2.elm.temp
     echo -n " ]" >> $2.elm.temp
-    cat $2.elm.temp | perl -0pe "s/\[\n ,/\[\n /g" | perl -pe "s/\0//g" > $2.elm
+    cat $2.elm.temp > $2.elm
     rm $2.elm.temp
     rm temp_list.txt
 }
 
 parse sfw Sfw
-parse nsfw Nsfw
+parse nsfw Nsfw
+
+echo "please manually remove comma in line 4"