both work

Author: ronen

pkg/octo-gitblit.edn

@@ -1,5 +1,5 @@
 {
- :workspace "/home/ronen/workspace/gitblit"
+ :workspace "/home/ronen/octo-development/gitblit"
  :user "admin"
  :password "vagrant"
  :url "https://192.168.121.22:8443/"
@@ -9,7 +9,7 @@
     }
 
    :rclone {
-    :dest "s3:narkisr-rclone/gitblit"
+    :dest "s3:octo-development/gitblit"
    }
   }
 }

src/octo/common/pull.clj

@@ -0,0 +1,72 @@
+(ns octo.common.pull
+  "pull backup into restore folder"
+  (:require
+    [clojure.string :refer (replace-first)]
+    [octo.common.push :refer [format-]]
+    [taoensso.timbre :as timbre]
+    [clojure.core.strint :refer  (<<)]
+    [octo.common :refer (safe lazy-mkdir rclone-sync files)]
+    [clojure.edn :refer (read-string)]
+    [me.raynes.fs :as fs]
+    [clojure.java.io :refer (file)]
+    [taoensso.timbre :as timbre]
+    [clojure.java.shell :refer [sh with-sh-dir]]))
+
+(timbre/refer-timbre)
+
+(defn parse [s] (.parse (java.text.SimpleDateFormat. format-) s))
+
+(defn parent
+  ([to id]
+    (<< "~{to}/zbackup/~{id}.zback"))
+  ([to id name]
+    (<< "~{to}/repos/~{id}/~{name}"))
+  )
+
+(defn validate [extracted to id]
+   (let [{:keys [total]} (read-string (slurp (<< "~{extracted}/check.edn")))
+          restored (count (files (<< "~{to}/repos/~{id}/")))]
+       (debug "validating restoration for" id)
+       (when-not (== total restored)
+         (throw (ex-info "Failed to restore found" {:found restored :expected total})))))
+
+(defn restore-bundles
+   [to id]
+   (info "to is" to)
+   (let [extracted (file to (.replace (.substring to 1) "pull" "sync") id "bundles")]
+     (info "restoring bundles from" extracted)
+     (doseq [bundle (filter #(and (.isFile %) (.endsWith (.getName %) "bundle")) (file-seq extracted))
+       :let [name (-> bundle (.getName) (.replace ".bundle" "")) target (parent to id name)]]
+       (when (.exists (file target)) (fs/delete-dir target))
+       (safe "git" "clone" (.getAbsolutePath bundle) target)
+       (with-sh-dir target (safe "git" "fsck" target)))
+     (validate extracted to id)
+     (fs/delete-dir extracted)))
+
+(defn zbackup-restore
+   "Restore an org/repo"
+   [to password id]
+   (let [base (<< "~(parent to id)/backups/")
+         backups (filter #(.isFile %) (file-seq (file base)))
+         latest (.getAbsolutePath (last (sort-by #(-> % (.getName) (.replace id "") parse) backups)))
+         target  (<< "~{to}/tars/~{id}.tar")]
+      (debug "zbackup restore")
+      (lazy-mkdir (<< "~{to}/tars"))
+      (safe "/bin/sh" "-c" (<< "zbackup --password-file ~{password} restore ~{latest} > ~{target}"))
+      (debug "extracting into" target)
+      (with-sh-dir to
+        (safe "tar" "xf" target))
+      (lazy-mkdir (<< "~{to}/repos"))
+      (restore-bundles to id)))
+
+(defn pull
+  [{:keys [zbackup rclone] :as m} to id]
+    (lazy-mkdir to)
+    (info "synching" id)
+    (rclone-sync  (<< "~(rclone :dest)/~{id}.zback") (parent to id))
+    (info "purging" (<< "~{to}/repos/~{id}"))
+    (fs/delete-dir (<< "~{to}/repos/~{id}"))
+    (info "restoring" id)
+    (when (empty? (.list (file (parent to id))))
+       (throw (ex-info "zbackup archive is empty, fix configuration, sync and push again." {:id id})))
+    (zbackup-restore to (zbackup :password-file) id))

src/octo/common/push.clj

@@ -0,0 +1,39 @@
+(ns octo.common.push
+  "push local backup remotely by using zbackup and rclone"
+  (:require
+    [me.raynes.fs :as fs :refer (base-name)]
+    [clojure.core.strint :refer  (<<)]
+    [octo.common :refer (safe lazy-mkdir rclone-sync)]
+    [clojure.java.io :refer (file)]
+    [taoensso.timbre :as timbre]
+    [clojure.java.shell :refer [sh with-sh-dir]]))
+
+(timbre/refer-timbre)
+
+(defn- run-backup
+  [source dest password]
+  (debug "source is" source)
+  (safe "/bin/sh" "-c" (<< "tar c ~{source} | zbackup backup --password-file ~{password} ~{dest}")))
+
+(defn- init
+  [password dest]
+  (safe "zbackup" "init" dest  "--password-file" password))
+
+(def format- "MM-dd-yyyy-HH-MM-ss")
+
+(defn now [] (.format  (java.text.SimpleDateFormat. format- ) (java.util.Date.)))
+
+(defn zbackup-backup
+   "Copy git backup into a zbackup based backup using backup id"
+   [password source dest id]
+   (when-not (.exists (file (<< "~{dest}/backups/")))
+     (init password dest))
+   (run-backup source  (<< "~{dest}/backups/~{id}-~(now)")  password))
+
+(defn push
+  [{:keys [zbackup rclone] :as m} source dest id]
+    (lazy-mkdir dest)
+    (info "zbackup backup" dest)
+    (zbackup-backup (zbackup :password-file) source dest id)
+    (info "rclone push" dest "into" (<< "~(rclone :dest)/~(base-name dest)"))
+    (rclone-sync dest (<< "~(rclone :dest)/~(base-name dest)")))

src/octo/common/synch.clj

@@ -14,7 +14,7 @@
      (delete-dir bundles)
      (doseq [[name url] (map f (filter (partial excluded? exclude) repos))
         :let [dest (<< "~{parent}/~{name}") op ((or options {}) (keyword name))]]
-         (info "synching" name)
+         (info "synching" name url dest op)
          (git/upclone url dest op)
          (info "mirrored" name )
          (git/bundle parent dest name)

src/octo/core.clj

@@ -20,7 +20,9 @@
       (println "octo backup" current ",latest version is" last-version "please upgrade")
       (println "octo backup" current))))
 
-(defn- run [[f {:keys [repos] :as m}]]
+(defn- run
+  "Running through repos list if exists else just pass m"
+  [[f {:keys [repos] :as m}]]
   (if repos
      (doseq [{:keys [user org] :as repo} repos]
        (info "processing:" (or user org))

src/octo/gitblit/pull.clj

@@ -0,0 +1,13 @@
+(ns octo.gitblit.pull
+  "pull backup into restore folder"
+  (:require
+    [octo.common.pull :as cp]
+    [taoensso.timbre :as timbre]
+    [clojure.core.strint :refer  (<<)]
+    ))
+
+(timbre/refer-timbre)
+
+(defn pull
+  [workspace m {:keys [user]}]
+    (cp/pull m (<< "~{workspace}/pull") user))

src/octo/gitblit/push.clj

@@ -1,48 +1,19 @@
 (ns octo.gitblit.push
   "push local backup remotly to the cloud"
  (:require
+    [octo.common.push :as cp]
     [clojure.core.strint :refer  (<<)]
-    [octo.common :refer (safe lazy-mkdir rclone-sync)]
-    [clojure.java.io :refer (file)]
     [taoensso.timbre :as timbre]
-    [clojure.java.shell :refer [sh with-sh-dir]]))
+    ))
 
 (timbre/refer-timbre)
 
 (defn parent
   ([workspace]
     (<< "~{workspace}/push"))
   ([workspace repo]
-      (<< "~(parent workspace)/~{repo}.zback")))
-
-(defn base [workspace repo]
-   (<< "~(parent workspace repo)/backups/~{repo}"))
-
-(defn- run-backup
-  [source dest password]
-  (safe "/bin/sh" "-c" (<< "tar c ~{source} | zbackup backup --password-file ~{password} ~{dest}")))
-
-(defn- init
-  [workspace password repo]
-  (safe "zbackup" "init" (parent workspace repo)  "--password-file" password))
-
-(def format- "MM-dd-yyyy-HH-MM-ss")
-
-(defn now [] (.format  (java.text.SimpleDateFormat. format- ) (java.util.Date.)))
-
-(defn zbackup-backup
-   "Copy git backup into a zbackup based backup"
-   [workspace password repo]
-   (let [base' (base workspace repo)]
-     (when-not (.exists (file (<< "~(parent workspace repo)/backups/")))
-       (init workspace password repo))
-     (run-backup (<< "~{workspace}/sync/bundles") (<< "~{base'}-~(now)")  password)))
+   (<< "~(parent workspace)/~{repo}.zback")))
 
 (defn push
-  [workspace {:keys [zbackup rclone] :as push} {:keys [user] :as m}]
-    (lazy-mkdir (parent workspace))
-    (info "zbackup backup" (<< "~{workspace}/sync/bundles"))
-    (zbackup-backup workspace (zbackup :password-file) user)
-    (info "rclone push up" user)
-    (rclone-sync (parent workspace user) (<< "~(rclone :dest)/~{user}.zback")))
-
+  [workspace m {:keys [user]}]
+  (cp/push m (<< "~{workspace}/sync/~{user}") (parent workspace user) user))

src/octo/gitblit/synch.clj

@@ -20,10 +20,11 @@
      (:body (client/get (<< "~{url}/rpc/?req=LIST_REPOSITORIES") basic))))
 
 (defn synch
-  [workspace auth m]
+  [workspace auth {:keys [user] :as m}]
    (cs/synch workspace auth (merge m {
-      :parent (<< "~{workspace}/sync") :bundles (<< "~{workspace}/sync/bundles")
-      :repos (repos m auth) :f (fn [[url {:keys [name]}]] [(replace name ".git" "") url])
+      :parent (<< "~{workspace}/sync/~{user}") :bundles (<< "~{workspace}/sync/~{user}/bundles")
+      :repos (repos m auth) 
+      :f (fn [[url {:keys [name]}]] [(replace name ".git" "") (.substring (str url) 1)])
      })))
 
 (comment

src/octo/github/pull.clj

@@ -1,72 +1,14 @@
 (ns octo.github.pull
   "pull backup into restore folder"
   (:require
-    [octo.github.push :refer [format-]]
+    [octo.common.pull :as cp]
     [taoensso.timbre :as timbre]
     [clojure.core.strint :refer  (<<)]
-    [octo.common :refer (safe lazy-mkdir rclone-sync files)]
-    [clojure.edn :refer (read-string)]
-    [me.raynes.fs :as fs]
-    [clojure.java.io :refer (file)]
-    [taoensso.timbre :as timbre]
-    [clojure.java.shell :refer [sh with-sh-dir]]))
+    ))
 
 (timbre/refer-timbre)
 
-(defn parse [s] (.parse (java.text.SimpleDateFormat. format-) s))
-
-(defn parent
-  ([workspace]
-   (<< "~{workspace}/pull") )
-  ([workspace repo]
-    (<< "~(parent workspace)/zbackup/~{repo}.zback"))
-  ([workspace repo name]
-    (<< "~(parent workspace)/repos/~{repo}/~{name}"))
-  )
-
-(defn validate [extracted workspace repo]
-   (let [{:keys [total]} (read-string (slurp (<< "~{extracted}/check.edn")))
-          restored (count (files (<< "~(parent workspace)/repos/~{repo}/")))]
-       (debug "validating restoration for" repo)
-       (when-not (== total restored)
-         (throw (ex-info "Failed to restore found" {:found restored :expected total})))))
-
-(defn restore-bundles
-   [workspace repo]
-   (let [extracted (file (parent workspace) (.substring workspace 1) "repos" repo "bundles")]
-     (info "restoring bundles from" extracted)
-     (doseq [bundle (filter #(and (.isFile %) (.endsWith (.getName %) "bundle")) (file-seq extracted))
-       :let [name (-> bundle (.getName) (.replace ".bundle" "")) target (parent workspace repo name)]]
-       (when (.exists (file target)) (fs/delete-dir target))
-       (safe "git" "clone" (.getAbsolutePath bundle) target)
-       (with-sh-dir target (safe "git" "fsck" target)))
-     (validate extracted workspace repo)
-     (fs/delete-dir extracted)))
-
-(defn zbackup-restore
-   "Restore an org/repo"
-   [workspace password repo]
-   (let [base (<< "~(parent workspace repo)/backups/")
-         backups (filter #(.isFile %) (file-seq (file base)))
-         latest (.getAbsolutePath (last (sort-by #(-> % (.getName) (.replace repo "") parse) backups)))
-         target  (<< "~(parent workspace)/tars/~{repo}.tar")]
-      (debug "zbackup restore")
-      (lazy-mkdir (<< "~(parent workspace)/tars"))
-      (safe "/bin/sh" "-c" (<< "zbackup --password-file ~{password} restore ~{latest} > ~{target}"))
-      (with-sh-dir (parent workspace)
-      (safe "tar" "xf" target))
-      (lazy-mkdir (<< "~(parent workspace)/repos"))
-      (restore-bundles workspace repo)))
-
 (defn pull
   [workspace {:keys [zbackup rclone] :as m} {:keys [org user]}]
-    (lazy-mkdir (parent workspace))
     (let [repo (or org user)]
-      (info "synching" repo)
-      (rclone-sync  (<< "~(rclone :dest)/~{repo}.zback") (parent workspace repo))
-      (info "purging" (<< "~(parent workspace)/repos/~{repo}"))
-      (fs/delete-dir (<< "~(parent workspace)/repos/~{repo}"))
-      (info "restoring" repo)
-      (when (empty? (.list (file (parent workspace repo))))
-        (throw (ex-info "zbackup archive empty fix configuration, sync and push" {:repo repo})))
-      (zbackup-restore workspace (zbackup :password-file) repo)))
+      (cp/pull m (<< "~{workspace}/pull") repo)))

src/octo/github/push.clj

@@ -1,11 +1,10 @@
 (ns octo.github.push
   "push local backup remotely by using zbackup and rclone"
   (:require
+    [octo.common.push :as cp]
     [clojure.core.strint :refer  (<<)]
-    [octo.common :refer (safe lazy-mkdir rclone-sync)]
-    [clojure.java.io :refer (file)]
     [taoensso.timbre :as timbre]
-    [clojure.java.shell :refer [sh with-sh-dir]]))
+    ))
 
 (timbre/refer-timbre)
 
@@ -15,35 +14,11 @@
   ([workspace repo]
       (<< "~(parent workspace)/~{repo}.zback")))
 
-(defn base [workspace repo]
-   (<< "~(parent workspace repo)/backups/~{repo}"))
-
-(defn- run-backup
-  [source dest password]
-  (safe "/bin/sh" "-c" (<< "tar c ~{source} | zbackup backup --password-file ~{password} ~{dest}")))
-
-(defn- init
-  [workspace password repo]
-  (safe "zbackup" "init" (parent workspace repo)  "--password-file" password))
-
-(def format- "MM-dd-yyyy-HH-MM-ss")
+(defn push
+  [workspace m {:keys [org user]}]
+   (let [repo (or org user)]
+    (cp/push m (<< "~{workspace}/sync/~{repo}/bundles") (parent workspace repo) repo)))
 
-(defn now [] (.format  (java.text.SimpleDateFormat. format- ) (java.util.Date.)))
 
-(defn zbackup-backup
-   "Copy git backup into a zbackup based backup"
-   [workspace password repo]
-   (let [base' (base workspace repo)]
-     (when-not (.exists (file (<< "~(parent workspace repo)/backups/")))
-       (init workspace password repo))
-     (run-backup (<< "~{workspace}/repos/~{repo}/bundles") (<< "~{base'}-~(now)")  password)))
 
-(defn push
-  [workspace {:keys [zbackup rclone] :as m} {:keys [org user]}]
-    (lazy-mkdir (parent workspace))
-    (let [repo (or org user)]
-      (info "zbackup backup" repo)
-      (zbackup-backup workspace (zbackup :password-file) repo)
-      (info "rclone push up" repo)
-      (rclone-sync (parent workspace repo) (<< "~(rclone :dest)/~{repo}.zback"))))