decode exit status

Author: technillogue

CHANGELOG.md

@@ -1,3 +1,4 @@
+0.1.7 decode downloader exit code. iterative merge tensors.
 0.1.6 algo arg for compression, respect LOAD_UNCOMPRESSED, fix vmsplice
 0.1.5 pass envvars to downloader + remotefile works with regional cache
 0.1.4 increase merge_tensors maxsize and make compression block size configurable

remotefile/main.go

@@ -93,12 +93,12 @@ func (b *DownloadBuffer) IsDone() bool {
 	return b.done && len(b.data) == 0
 }
 
-func downloadToBuffer(buf *DownloadBuffer) {
+func downloadToBuffer(buf *DownloadBuffer) int64 {
 	startTime := time.Now()
 	resp, err := client.Get(buf.url)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Error fetching %s: %v\n", buf.url, err)
-		return
+		return 0
 	}
 	defer resp.Body.Close()
 
@@ -118,11 +118,11 @@ func downloadToBuffer(buf *DownloadBuffer) {
 			throughput := float64(resp.ContentLength) / elapsed.Seconds() / 1024 / 1024
 			fmt.Fprintf(os.Stderr, "Downloaded %s in %s (%.2f MB/s)\n", resp.Request.URL, elapsed, throughput)
 			buf.chunkPool.Put(chunk)
-			return
+			return resp.ContentLength
 		}
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Error reading from %s: %v\n", resp.Request.URL, err)
-			return
+			return 0
 		}
 	}
 }
@@ -177,15 +177,20 @@ func main() {
 	}
 	bufChan := make(chan *DownloadBuffer)
 	go func(bufChan chan *DownloadBuffer) {
+		total_size := int64(0)
+		start := time.Now()
 		for _, url := range os.Args[1:] {
 			// ignore curl args
 			if url == "-s" || url == "-v" {
 				continue
 			}
 			buf := NewBuffer(url)
 			bufChan <- buf
-			downloadToBuffer(buf)
+			total_size += downloadToBuffer(buf)
 		}
+		elapsed := time.Since(start)
+		throughput := float64(total_size) / elapsed.Seconds() / 1024 / 1024
+		fmt.Fprintf(os.Stderr, "Overall downloaded %d MB in %s (%.2f MB/s)\n", total_size/1024/1024, elapsed, throughput)
 		close(bufChan)
 	}(bufChan)
 	for buf := range bufChan {

src/main.cpp

@@ -242,7 +242,15 @@ class DownloadProc {
         throw std::runtime_error("Failed to spawn curl subprocess. Error code: " + std::to_string(retcode));
       int status;
       waitpid(pid, &status, 0);
-      throw std::runtime_error("Failed to spawn curl subprocess. Error code: " + std::to_string(retcode) + ", exit code: " + std::to_string(status));
+      // decode status
+      if (WIFEXITED(status))
+        throw std::runtime_error("Failed to spawn curl subprocess. Error code: " + std::to_string(retcode) + ", exit code: " + std::to_string(WEXITSTATUS(status)));
+      else if (WIFSIGNALED(status))
+        throw std::runtime_error("Failed to spawn curl subprocess. Error code: " + std::to_string(retcode) + ", exit signal: " + std::to_string(WTERMSIG(status)));
+      else if (WIFSTOPPED(status))
+        throw std::runtime_error("Failed to spawn curl subprocess. Error code: " + std::to_string(retcode) + ", stop signal: " + std::to_string(WSTOPSIG(status)));
+      else
+        throw std::runtime_error("Failed to spawn curl subprocess. Error code: " + std::to_string(retcode) + ", unknown status: " + std::to_string(status));
     }
     // close write end of the pipe
     close(pipefd[1]);
@@ -256,7 +264,7 @@ class DownloadProc {
     kill(pid, SIGTERM);
     int status;
     waitpid(pid, &status, 0);
-    int code = ((WIFEXITED(status) ? WEXITSTATUS(status) : (WIFSIGNALED(status) ? -WTERMSIG(status) : 0)))
+    int code = ((WIFEXITED(status) ? WEXITSTATUS(status) : (WIFSIGNALED(status) ? -WTERMSIG(status) : 0)));
     log("killed downloader process. exit code: " + std::to_string(code));
   }
 

tools/build.sh

@@ -1,7 +1,7 @@
 set -o xtrace
 set -o pipefail
 set -o errexit
-export VERSION="0.1.6"
+export VERSION="0.1.7"
 # worked with auditwheel for manywheel torch, but not freshly compiled recent-glibc torch
 # nyacomp-$VERSION-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
 WHEEL="nyacomp-$VERSION-cp311-cp311-linux_x86_64.whl"