Merge pull request #4 from ipfs/importer

WIP - Importer

Author: daviddias

.gitignore

@@ -1,3 +1,7 @@
+
+.DS_Store
+tests/repo-tests*
+
 # Logs
 logs
 *.log

package.json

@@ -5,8 +5,8 @@
   "main": "src/index.js",
   "scripts": {
     "lint": "standard",
-    "coverage": "istanbul cover --print both -- _mocha tests/test-*.js",
-    "test": "mocha tests/test-*.js"
+    "coverage": "istanbul cover --print both -- _mocha tests/index.js",
+    "test": "mocha tests/index.js"
   },
   "repository": {
     "type": "git",
@@ -22,14 +22,23 @@
   },
   "homepage": "https://github.com/diasdavid/js-ipfs-data-importing#readme",
   "devDependencies": {
+    "bs58": "^3.0.0",
     "chai": "^3.4.1",
+    "fs-blob-store": "^5.2.1",
+    "ipfs-repo": "^0.5.0",
     "istanbul": "^0.4.1",
     "mocha": "^2.3.4",
+    "ncp": "^2.0.0",
     "pre-commit": "^1.1.2",
+    "rimraf": "^2.5.1",
     "standard": "^5.4.1"
   },
   "dependencies": {
+    "async": "^1.5.2",
     "debug": "^2.2.0",
+    "ipfs-blocks": "^0.1.0",
+    "ipfs-merkle-dag": "^0.2.1",
+    "ipfs-unixfs": "^0.1.0",
     "through2": "^2.0.0"
   }
 }

src/chunker-fixed-size.js

@@ -20,9 +20,9 @@ function FixedSizeChunker (size) {
     }
 
     function slice () {
-      var chunk = new Buffer(size)
-      var newBuf = new Buffer(buf.length - size)
-      buf.copy(chunk, 0, 0, size - 1)
+      var chunk = new Buffer(size, 'binary')
+      var newBuf = new Buffer(buf.length - size, 'binary')
+      buf.copy(chunk, 0, 0, size)
       buf.copy(newBuf, 0, size - 1, buf.length - size)
       buf = newBuf
       that.push(chunk)

src/index.js

@@ -1,27 +1,173 @@
-var debug = require('debug')
-var log = debug('importer')
+const debug = require('debug')
+const log = debug('importer')
 log.err = debug('importer:error')
-var fs = require('fs')
+const fs = require('fs')
+const mDAG = require('ipfs-merkle-dag')
+const FixedSizeChunker = require('./chunker-fixed-size')
+const through2 = require('through2')
+const UnixFS = require('ipfs-unixfs')
+const async = require('async')
 
 exports = module.exports
 
+const CHUNK_SIZE = 262144
+
 // Use a layout + chunkers to convert a directory (or file) to the layout format
-exports.import = function (options, callback) {
-  // options.store -> where to write stuff (typically js-ipfs-repo.datastore, which impls blob-store)
-  // options.path -> where to
+exports.import = (options, callback) => {
+  // options.path -> what to import
   // options.recursive -> follow dirs
   // options.chunkers -> obj with chunkers to each type of data, { default: dumb-chunker }
+  // options.dag-service-> instance of block service
+  const dagService = options.dagService
 
-  var pathStats = fs.statSync(options.path)
-  if (pathStats.isFile()) {
+  const stats = fs.statSync(options.path)
+  if (stats.isFile()) {
+    fileImporter(options.path, callback)
+  } else if (stats.isDirectory() && options.recursive) {
+    dirImporter(options.path, callback)
+  } else {
+    return callback(new Error('recursive must be true to add a directory'))
+  }
 
+  function fileImporter (path, callback) {
+    const stats = fs.statSync(path)
+    if (stats.size > CHUNK_SIZE) {
+      const links = [] // { Hash: , Size: , Name: }
 
-  } else if (pathStats.isDir() && options.recursive) {
+      fs.createReadStream(path)
+        .pipe(new FixedSizeChunker(CHUNK_SIZE))
+        .pipe(through2((chunk, enc, cb) => {
+          // TODO: check if this is right (I believe it should be type 'raw'
+          // https://github.com/ipfs/go-ipfs/issues/2331
+          const raw = new UnixFS('file', chunk)
 
-  } else {
-    return callback(new Error('recursive must be true to add a directory'))
+          const node = new mDAG.DAGNode(raw.marshal())
+
+          dagService.add(node, function (err) {
+            if (err) {
+              return log.err(err)
+            }
+            links.push({
+              Hash: node.multihash(),
+              Size: node.size(),
+              leafSize: raw.fileSize(),
+              Name: ''
+            })
+
+            cb()
+          })
+        }, (cb) => {
+          const file = new UnixFS('file')
+          const parentNode = new mDAG.DAGNode()
+          links.forEach((l) => {
+            file.addBlockSize(l.leafSize)
+            const link = new mDAG.DAGLink(l.Name, l.Size, l.Hash)
+            parentNode.addRawLink(link)
+          })
+
+          parentNode.data = file.marshal()
+          dagService.add(parentNode, (err) => {
+            if (err) {
+              return log.err(err)
+            }
+
+            const pathSplit = path.split('/')
+            const fileName = pathSplit[pathSplit.length - 1]
+
+            callback(null, {
+              Hash: parentNode.multihash(),
+              Size: parentNode.size(),
+              Name: fileName
+            }) && cb()
+          })
+        }))
+    } else {
+      // create just one file node with the data directly
+      const fileUnixFS = new UnixFS('file', fs.readFileSync(path))
+      const fileNode = new mDAG.DAGNode(fileUnixFS.marshal())
+
+      dagService.add(fileNode, (err) => {
+        if (err) {
+          return log.err(err)
+        }
+
+        const split = path.split('/')
+        const fileName = split[split.length - 1]
+
+        callback(null, {
+          Hash: fileNode.multihash(),
+          Size: fileNode.size(),
+          Name: fileName
+        })
+      })
+    }
+  }
+
+  function dirImporter (path, callback) {
+    const files = fs.readdirSync(path)
+    const dirUnixFS = new UnixFS('directory')
+    const dirNode = new mDAG.DAGNode()
+
+    if (files.length === 0) {
+      dirNode.data = dirUnixFS.marshal()
+      dagService.add(dirNode, (err) => {
+        if (err) {
+          return callback(err)
+        }
+
+        const split = path.split('/')
+        const dirName = split[split.length - 1]
+
+        callback(null, {
+          Hash: dirNode.multihash(),
+          Size: dirNode.size(),
+          Name: dirName
+        })
+      })
+      return
+    }
+
+    async.map(
+      files,
+      (file, cb) => {
+        const filePath = path + '/' + file
+        const stats = fs.statSync(filePath)
+        if (stats.isFile()) {
+          return fileImporter(filePath, cb)
+        } if (stats.isDirectory()) {
+          return dirImporter(filePath, cb)
+        } else {
+          return callback(new Error('Found a weird file' + path + file))
+        }
+      },
+      (err, results) => {
+        if (err) {
+          return callback(err)
+        }
+        results.forEach((result) => {
+          dirNode.addRawLink(new mDAG.DAGLink(result.Name, result.Size, result.Hash))
+          dirNode.data = dirUnixFS.marshal()
+          dagService.add(dirNode, (err) => {
+            if (err) {
+              return callback(err)
+            }
+
+            const split = path.split('/')
+            const dirName = split[split.length - 1]
+
+            callback(null, {
+              Hash: dirNode.multihash(),
+              Size: dirNode.size(),
+              Name: dirName
+            })
+          })
+        })
+      })
   }
+  // function bufferImporter (path) {}
+  // function streamImporter (path) {}
 }
 
 exports.export = function () {
+  // export into files by hash
 }

src/layout-merkledag.js

@@ -0,0 +1,43 @@
+/* globals describe, before, after */
+
+'use strict'
+
+const fs = require('fs')
+const ncp = require('ncp').ncp
+const rimraf = require('rimraf')
+const expect = require('chai').expect
+
+describe('core', () => {
+  const repoExample = process.cwd() + '/tests/repo-example'
+  const repoTests = process.cwd() + '/tests/repo-tests' + Date.now()
+
+  before(done => {
+    ncp(repoExample, repoTests, err => {
+      process.env.IPFS_PATH = repoTests
+      expect(err).to.equal(null)
+      done()
+    })
+  })
+
+  after(done => {
+    rimraf(repoTests, err => {
+      expect(err).to.equal(null)
+      done()
+    })
+  })
+
+  const tests = fs.readdirSync(__dirname)
+  tests.filter(file => {
+    if (file === 'index.js' ||
+        file === 'browser.js' ||
+        file === 'test-data' ||
+        file === 'repo-example' ||
+        file.indexOf('repo-tests') > -1) {
+      return false
+    } else {
+      return true
+    }
+  }).forEach(file => {
+    require('./' + file)
+  })
+})

tests/index.js

@@ -0,0 +1,28 @@
+
+��Hello and Welcome to IPFS!
+
+██╗██████╗ ███████╗███████╗
+██║██╔══██╗██╔════╝██╔════╝
+██║██████╔╝█████╗  ███████╗
+██║██╔═══╝ ██╔══╝  ╚════██║
+██║██║     ██║     ███████║
+╚═╝╚═╝     ╚═╝     ╚══════╝
+
+If you're seeing this, you have successfully installed
+IPFS and are now interfacing with the ipfs merkledag!
+
+ -------------------------------------------------------
+| Warning:                                              |
+|   This is alpha software. Use at your own discretion! |
+|   Much is missing or lacking polish. There are bugs.  |
+|   Not yet secure. Read the security notes for more.   |
+ -------------------------------------------------------
+
+Check out some of the other files in this directory:
+
+  ./about
+  ./help
+  ./quick-start     <-- usage examples
+  ./readme          <-- this file
+  ./security-notes
+�

tests/repo-example/blocks/1220120f/1220120f6af601d46e10b2d2e11ed71c55d25f3042c22501e41d1246e7a1e9d3d8ec.data

@@ -0,0 +1,4 @@
+5
+" ���׾F�_�uؔ�l��z�S?��|ڲ��Pc@js-ipfs-repo�
+
+