Changes while I wrote an indexer

.github/workflows/rust.yml

@@ -0,0 +1,35 @@
+name: Rust
+
+on: [push]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: nightly
+          components: clippy
+          override: true
+      - name: Build
+        run: cargo build --verbose
+      - name: Run tests
+        run: cargo test --verbose
+      - name: Cache cargo registry
+        uses: actions/cache@v1
+        with:
+          path: ~/.cargo/registry
+          key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
+      - name: Cache cargo index
+        uses: actions/cache@v1
+        with:
+          path: ~/.cargo/git
+          key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.lock') }}
+      - name: Cache cargo build
+        uses: actions/cache@v1
+        with:
+          path: target
+          key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('**/Cargo.lock') }}

.gitignore

@@ -1,3 +1,6 @@
 /target
 /Cargo.lock
 .DS_Store
+*iml
+*ipr
+*iws

Cargo.toml

@@ -7,11 +7,20 @@ repository = "https://github.com/mattnenterprise/rust-nntp"
 description = "NNTP client for Rust"
 readme = "README.md"
 license = "MIT OR Apache-2.0"
+edition = "2018"
 
-[lib]
-name = "nntp"
-path = "src/nntp.rs"
-
-[[bin]]
-name = "example"
-path = "example.rs"
+[dependencies]
+bufstream = "0.1"
+prettytable-rs = "0.8.0"
+native-tls = "0.2.3"
+pretty-bytes = "0.2.2"
+itertools = "0.8.0"
+log = "0.4.6"
+env_logger = "0.7.1"
+yenc = "0.1.1"
+flate2 = "1.0.7"
+elasticsearch = "7.5.2-alpha.1"
+serde = { version = "1.0.104", features = ["derive"] }
+serde_json = "1.0"
+tokio = { version = "0.2.11", features = ["full"] }
+bytes = "0.5.4"

docker-compose.yml

@@ -0,0 +1,80 @@
+version: '2.2'
+services:
+  kibana:
+    image: docker.elastic.co/kibana/kibana-oss:7.5.2
+    container_name: kibana
+    environment:
+      SERVER_NAME: kibana.example.org
+      ELASTICSEARCH_HOSTS: http://es01:9200
+    ports:
+      - 5601:5601
+    networks:
+      - elastic
+  es01:
+    image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.5.2
+    container_name: es01
+    environment:
+      - node.name=es01
+      - cluster.name=es-docker-cluster
+      - discovery.seed_hosts=es02,es03
+      - cluster.initial_master_nodes=es01,es02,es03
+      - bootstrap.memory_lock=true
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    volumes:
+      - data01:/usr/share/elasticsearch/data
+    ports:
+      - 9200:9200
+    networks:
+      - elastic
+  es02:
+    image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.5.2
+    container_name: es02
+    environment:
+      - node.name=es02
+      - cluster.name=es-docker-cluster
+      - discovery.seed_hosts=es01,es03
+      - cluster.initial_master_nodes=es01,es02,es03
+      - bootstrap.memory_lock=true
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    volumes:
+      - data02:/usr/share/elasticsearch/data
+    networks:
+      - elastic
+  es03:
+    image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.5.2
+    container_name: es03
+    environment:
+      - node.name=es03
+      - cluster.name=es-docker-cluster
+      - discovery.seed_hosts=es01,es02
+      - cluster.initial_master_nodes=es01,es02,es03
+      - bootstrap.memory_lock=true
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    volumes:
+      - data03:/usr/share/elasticsearch/data
+    networks:
+      - elastic
+
+volumes:
+  data01:
+    driver: local
+  data02:
+    driver: local
+  data03:
+    driver: local
+
+networks:
+  elastic:
+    driver: bridge

examples/fetch_articles.rs

@@ -0,0 +1,51 @@
+extern crate nntp;
+#[macro_use]
+extern crate prettytable;
+
+use nntp::{NNTPStream, ParsedArticle};
+use prettytable::Table;
+
+fn main() -> Result<(), std::io::Error> {
+    let buf_stream = nntp::tls_buf_stream("nntp.aioe.org", 119)?;
+    let mut nntp_stream = NNTPStream::connect(buf_stream)?;
+
+    let lines = nntp_stream.capabilities()?;
+    for line in lines.iter() {
+        print!("{}", line);
+    }
+
+    let groups = nntp_stream.list()?;
+    for group in groups.iter() {
+        println!(
+            "Name: {}, High: {}, Low: {}, Status: {}",
+            group.name, group.high, group.low, group.status
+        )
+    }
+
+    nntp_stream.group("comp.sys.raspberry-pi")?;
+
+    let article = nntp_stream.article_by_number(20000)?;
+    let ParsedArticle { headers, body, .. } = article.parse()?;
+    let mut table = Table::new();
+    for (key, value) in headers.headers.iter() {
+        table.add_row(row![key, value]);
+    }
+    table.printstd();
+
+    println!("==== BODY ====");
+    println!("{}", std::str::from_utf8(body).expect("valid utf8"));
+
+    let article = nntp_stream.article_by_id("<a55pbedl7rf6sr0h1d9bf37q5qpj0rgn5j@4ax.com>")?;
+    let ParsedArticle { headers, body, .. } = article.parse()?;
+    let mut table = Table::new();
+    for (key, value) in headers.headers.iter() {
+        table.add_row(row![key, value]);
+    }
+    table.printstd();
+
+    println!("==== BODY ====");
+    println!("{}", std::str::from_utf8(body).expect("valid utf8"));
+
+    println!("COMMAND: quit");
+    nntp_stream.quit()
+}

examples/index_group.rs

@@ -0,0 +1,92 @@
+extern crate nntp;
+#[allow(unused_imports)]
+#[macro_use]
+extern crate prettytable;
+extern crate bufstream;
+
+#[allow(unused_imports)]
+use std::collections::HashMap;
+
+use nntp::{NNTPStream, NewsGroup};
+#[allow(unused_imports)]
+use prettytable::Table;
+
+use bufstream::BufStream;
+
+fn main() -> Result<(), std::io::Error> {
+    let tcp_stream = std::net::TcpStream::connect(("us.newsgroupdirect.com", 563))?;
+
+    let connector = native_tls::TlsConnector::new().unwrap();
+    let stream = connector
+        .connect("us.newsgroupdirect.com", tcp_stream)
+        .map_err(|_x| std::io::Error::new(std::io::ErrorKind::Other, "tls failed"))?;
+    let stream = BufStream::new(stream);
+    let mut nntp_stream = NNTPStream::connect(stream)?;
+
+    //    let GROUP = "comp.sys.raspberry-pi";
+    let group = "alt.binaries.warez";
+
+    use std::env;
+    let envmap: HashMap<String, String> = env::vars().collect();
+    nntp_stream.authinfo_user(envmap.get("NEWSGROUP_USER").expect("newsgroup user"))?;
+    nntp_stream.authinfo_pass(envmap.get("NEWSGROUP_PASS").expect("newsgroup pass"))?;
+
+    let _cap = nntp_stream.capabilities()?;
+    // panic!("cap (after AUTH): {:#?}", cap);
+
+    let groups = nntp_stream.list()?;
+    let groups_by_name: HashMap<&str, &NewsGroup> =
+        groups.iter().map(|x| (&x.name[..], x)).collect();
+
+    let g = *groups_by_name.get(group).unwrap();
+    println!("the g: {:#?}", g);
+
+    let mut t = Table::new();
+    for (name, _) in groups_by_name.iter() {
+        t.add_row(row![name]);
+    }
+    t.printstd();
+    //
+    nntp_stream.group(group)?;
+    let _article = nntp_stream.article()?;
+    let _stat = nntp_stream.stat()?;
+    //    let article = nntp_stream.article_by_number(3269684000).unwrap();
+    //    let article = article.parse()?;
+    //    let mut t = Table::new();
+    //    for (k,v) in article.headers.headers.iter() {
+    //        t.add_row(row![k,v]);
+    //    }
+    //    t.printstd();
+    //    println!("body size: {}", article.body.len());
+
+    let mut failure_count = 0;
+
+    println!("going through the group");
+    while nntp_stream.next().is_ok() {
+        match nntp_stream.head() {
+            Ok(headers) => {
+                println!("parsing headers");
+                let parsed = headers.parse()?;
+                println!("{:?}", parsed);
+                println!("code: {}, message: {}", parsed.code, parsed.message);
+                println!("bleep: {:?}", parsed.headers[0])
+            }
+            Err(e) => {
+                failure_count += 1;
+                if failure_count > 3 {
+                    panic!("too many failures");
+                }
+                println!("error {}...", e);
+                continue;
+            }
+        }
+    }
+
+    let last_response = nntp_stream.next()?;
+    println!("response: {}", last_response);
+    let _ = nntp_stream.head()?;
+    //    panic!("got whatever\n{:#?}", whatever);
+
+    println!("COMMAND: quit");
+    nntp_stream.quit()
+}