Merge remote-tracking branch 'upstream/master' into parallelize

Author: Inventitech

.gitignore

@@ -3,6 +3,6 @@ npm-debug.log
 
 node_modules/
 assets/
-repos/
-gitlogg.json
-gitlogg.tmp
+_repos/
+_output/
+_tmp/

README.md

@@ -28,7 +28,7 @@
 ![Error 002](https://raw.githubusercontent.com/dreamyguy/gitlogg/master/docs/error-002.png "'Error 002' message as on release v0.1.3")
 > **Øh nøes!** The path to the folder containing all repositories *exists, but is empty!*
 
-![Success!](https://raw.githubusercontent.com/dreamyguy/gitlogg/master/docs/success.png "Success messages as on release v0.1.3")
+![Success!](https://raw.githubusercontent.com/dreamyguy/gitlogg/master/docs/success.png "Success messages as on release v0.1.6")
 > **Success!** `JSON` parsed, based on **9** different repositories with a total of **25,537** commits.
 
 Note that I've included two huge repos _(*react* & *react-native*, that have 7,813 & 10,065 commits respectively at the time of this writting)_ for the sake of demonstration. The resulting parsed `JSON` file has 715,040 lines. All that done in less than 25 seconds.
@@ -41,13 +41,17 @@ _I have successfully compiled **`470`** repositories at once_ (all repos under t
 * Commits processed: `118,117`
 * Parsed `JSON` file, lines: `3,307,280`
 
-## Requirementss
+## Getting started
 
-[NodeJS][2] and [BabelJS][3].
+**Gitlogg** requires [NodeJS][2] and [BabelJS][3].
 
 1. Install `NodeJS` (visit [their page][2] to find the right install for your system).
-2. Install `BabelJS` globally by running `npm install babel-cli -g`. One can also choose to install it locally by simply running `npm install babel-cli`, but in most cases it is smarter to install `-cli` packages globally.
-3. Run `npm install` to install the remaining dependencies.
+2. Run `npm run setup`. That will:
+
+* Install `BabelJS` globally by running `npm install babel-cli -g`.
+* Install all the local dependencies, through `npm install`.
+* Create the directory in which all repos to be parsed to `JSON` will be at (only on **Simple Mode**).
+* Create the directories expected by the scripts that output files.
 
 ## The `JSON` output
 
@@ -152,56 +156,68 @@ There are two modes and they are basically the same, except that the **Simple Mo
 
 To simplify the generation process to a point that no configuration is required, follow this directory structure:
 
-    .
-    ├── gitlogg
+    gitlogg/          <== This repository's root
+    ├── scripts/
     │   ├── gitlogg-generate-log.sh
     │   ├── gitlogg-parse-json.js
     │   └── gitlogg.sh
-    └── repos         <== place/keep your repositories under the folder "repos"
+    └── _repos/       <== Copy/place/keep your repositories under the folder "_repos/"
         ├── repo1
         ├── repo2
         ├── repo3
         └── repo4
 
-1. Copy the `gitlogg` folder and all its content to the indicated relative path to your local repositories (shown above).
-
-2. Navigate to the `gitlogg` directory:
+1. Copy the all the repositories you wish to parse to `JSON` to the `_repos/` folder, as shown above.
 
-        $ cd path/to/the/folder/in/your/system/gitlogg/
+2. Granted that you are within the `gitlogg` folder (this repo's root), run:
 
-3. Run it:
-
-        $ ./gitlogg.sh
+        $ npm run gitlogg
 
 #### Advanced Mode
 
 To generate the `JSON` file based on repositories in any other location, you'll have to define the path to the folder that contains all your repositories.
 
-1. Copy the `gitlogg` folder and all its content to a folder of your preference, it really doesn't matter where it is.
-
-2. Open [`gitlogg-generate-log.sh`](https://github.com/dreamyguy/gitlogg/blob/master/gitlogg/gitlogg-generate-log.sh#L17) with an editor of your choice and edit the `yourpath` variable:
+1. Open [`gitlogg-generate-log.sh`](https://github.com/dreamyguy/gitlogg/blob/master/scripts/gitlogg-generate-log.sh#L4) with an editor of your choice and edit the `yourpath` variable:
 
         # define the absolute path to the directory that contains all your repositories
-        yourpath=~/path/to/directory/that/contains/all/your/repositories/
+        yourpath=/absolute/system/path/to/directory/that/contains/all/your/repositories/
 
-3. Navigate to the `gitlogg` directory:
+_**Tip:** drag the folder that contain your repositories to a terminal window, and you'll get the absolute system path to that folder._
 
-        $ cd path/to/the/folder/in/your/system/gitlogg/
+2. Granted that you are within the `gitlogg` folder (this repo's root), run:
 
-4. Run it:
-
-        $ ./gitlogg.sh
+        $ npm run gitlogg
 
 ## The parsed `JSON` file
 
-> _Two files will be generated at the `gitlogg` folder: `gitlogg.tmp` and `gitlogg.json`._
+> Two files will be generated when running `npm run gitlogg`: **`_tmp/gitlogg.tmp`** and **`_output/gitlogg.json`**.
+
+    gitlogg/                <== This repository's root
+    ├── scripts/
+    │   ├── gitlogg-generate-log.sh
+    │   ├── gitlogg-parse-json.js
+    │   └── gitlogg.sh
+    ├── _output/
+    │   └── gitlogg.json    <== The parsed 'JSON', what we're all after. It's parsed from 'gitlogg.tmp'
+    └── _tmp/
+        └── gitlogg.tmp     <== The processed 'git log'
 
 Two files were necessary because of the nature of the script, that loops through all subdirectories and outputs the `git log` for all valid `git` repositories. Once that loop is done, a valid `JSON` file (`gitlogg.json`) is generated out of `gitlogg.tmp`.
 
 `gitlogg.tmp` is just a temporary file from which `gitlogg.json` bases itself on. In case the parsing fails `gitlogg.tmp` can come in handy for debugging.
 
 ## Further Notes
 
+#### Debugging
+
+I've created error messages with suggested solutions, to help you get past the most common issues.
+
+However, `git log`'s output can break while it's being processed. That's most certainly caused by fields that allow user input, like _commit messages_. These fields may contain characters (like `\r`) that crash with those reserved for the generation of `gitlogg.tmp`, namely `\n`.
+
+Efforts have been made to mitigate errors by sanitizing characters that have caused errors before, but it might still happen in some edge cases. If it does happen, have a look at the generated `gitlogg.tmp` and see if the expected structure (which is obvious) breaks. Once you have identified the line, have a closer look at the commit and look for an unusual character.
+
+Post an issue with a link to a _gist_ containing your broken `gitlogg.tmp` and I will try to reproduce the error.
+
 #### Documentation
 
 Documentation is done either by:
@@ -227,22 +243,32 @@ It's certainly not harmful to your repositories and it won't change any data in
 
 #### Release History
 
-* 2016-11-19   [v0.1.6](https://github.com/dreamyguy/gitlogg/tree/v0.1.6)
+* 2016-11-21   [v0.1.7](https://github.com/dreamyguy/gitlogg/tree/v0.1.7) - [View Changes](https://github.com/dreamyguy/gitlogg/compare/v0.1.6...v0.1.7)
+  * Better readability for 'Release History'
+  * Correct url to logo, so it also renders outside Github
+  * Rename sub-folder 'gitlogg' to 'scripts' to avoid confusion
+  * Simplify initial setup and running of 'gitlogg'
+  * Set vars instead of hardcoding values
+  * Separate scripts from output files
+  * Introduce 'Debugging' as a 'Further Notes' item
+  * Tip on how to get the absolute system path to a directory
+  * Introduce 'View Changes' links under 'Release History'
+* 2016-11-19   [v0.1.6](https://github.com/dreamyguy/gitlogg/tree/v0.1.6) - [View Changes](https://github.com/dreamyguy/gitlogg/compare/v0.1.5...v0.1.6)
   * Introduce `commit_nr`, a commit count within each repo
   * Show how many repos are about to be processed on console
   * Show what repo is being processed on console
   * Replace carriage return with space
-* 2016-06-12   [v0.1.5](https://github.com/dreamyguy/gitlogg/tree/v0.1.5)
+* 2016-06-12   [v0.1.5](https://github.com/dreamyguy/gitlogg/tree/v0.1.5) - [View Changes](https://github.com/dreamyguy/gitlogg/compare/v0.1.4...v0.1.5)
   * Introduce logo
   * Correct wrong reference to 'yourpath'
   * Output numbers instead of strings
-* 2016-05-23   [v0.1.4](https://github.com/dreamyguy/gitlogg/tree/v0.1.4)
+* 2016-05-23   [v0.1.4](https://github.com/dreamyguy/gitlogg/tree/v0.1.4) - [View Changes](https://github.com/dreamyguy/gitlogg/compare/v0.1.3...v0.1.4)
   * Fix a bug that would break the output in some rare cases
-* 2016-05-21   [v0.1.3](https://github.com/dreamyguy/gitlogg/tree/v0.1.3)
+* 2016-05-21   [v0.1.3](https://github.com/dreamyguy/gitlogg/tree/v0.1.3) - [View Changes](https://github.com/dreamyguy/gitlogg/compare/v0.1.2...v0.1.3)
   * Even better error handling
-* 2016-05-21   [v0.1.2](https://github.com/dreamyguy/gitlogg/tree/v0.1.2)
+* 2016-05-21   [v0.1.2](https://github.com/dreamyguy/gitlogg/tree/v0.1.2) - [View Changes](https://github.com/dreamyguy/gitlogg/compare/v0.1.1...v0.1.2)
   * Better error handling
-* 2016-05-21   [v0.1.1](https://github.com/dreamyguy/gitlogg/tree/v0.1.1)
+* 2016-05-21   [v0.1.1](https://github.com/dreamyguy/gitlogg/tree/v0.1.1) - [View Changes](https://github.com/dreamyguy/gitlogg/compare/v0.1.0...v0.1.1)
   * The 'gitlogg' release, the node-based JSON generation
 * 2016-05-20   [v0.1.0](https://github.com/dreamyguy/gitlogg/tree/v0.1.0)
   * The 'git-log-to-json' release, now considered legacy

package.json

@@ -1,6 +1,6 @@
 {
   "name": "gitlogg",
-  "version": "0.1.6",
+  "version": "0.1.7",
   "description": "Parse the 'git log' of one or several 'git' repositories into a sanitised and distributable 'JSON' file",
   "keywords": [
     "gitlog",
@@ -26,7 +26,11 @@
     "babel-preset-es2015": "^6.9.0",
     "chalk": "^1.1.3"
   },
+  "scripts": {
+    "setup": "npm install babel-cli -g && npm install && mkdir -p _output && mkdir -p _repos && mkdir -p _tmp",
+    "gitlogg": "./scripts/gitlogg.sh"
+  },
   "engines": {
-    "node": ">=4.0.3"
+    "node": ">=4.3.0"
   }
 }

gitlogg/colors.sh renamed to scripts/colors.sh

@@ -1,7 +1,26 @@
 #!/bin/bash
 
+my_dir="$(dirname "$0")"
+cd $my_dir
+
 source "colors.sh"
 
+cd ..
+pwd
+
+# define the absolute path to the directory that contains all your repositories.
+yourpath='./_repos/'
+
+# define temporary 'git log' output file that will be parsed to 'json'
+tempOutputFile='_tmp/gitlogg.tmp'
+
+# name and path to this very script, for output message purposes
+thisFile='./scripts/gitlogg-generate-log.sh'
+
+# define path to 'json' parser
+jsonParser='./scripts/gitlogg-parse-json.js'
+
+
 # Display system usage and exit
 usage()
 {
@@ -22,9 +41,6 @@ test $NUM_THREADS -lt 1 && NUM_THREADS=1
 test "$1" = "-n" && NUM_THREADS=$2
 echo -e "${Blu}Info: Calculating in $NUM_THREADS thread(s)${RCol}"
 
-# define the absolute path to the directory that contains all your repositories.
-yourpath=../repos/
-
 # ensure there's always a '/' at the end of the 'yourpath' variable, since its value can be changed by user.
 case "$yourpath" in
   */)
@@ -67,15 +83,15 @@ SECONDS=0
 if [ -d "${yourpathSanitized}" ] && [ "$(ls $yourpathSanitized)" ]; then
   echo -e "${Yel}Generating ${Pur}git log ${Yel}for ${reporef} located at ${Red}'${thepath}'${Yel}. ${Blu}This might take a while!${RCol}"
   dirs=$(ls -d $thepath)
-  echo $dirs | xargs -n 1 -P $NUM_THREADS ./output-intermediate-gitlog.sh > gitlogg.tmp
-  echo -e "${Gre}The file ${Blu}./gitlogg.tmp ${Gre}generated in${RCol}: ${SECONDS}s" &&
-  babel gitlogg-parse-json.js | node        # only parse JSON if we have a source to parse it from
+  echo $dirs | xargs -n 1 -P $NUM_THREADS ./output-intermediate-gitlog.sh > ${tempOutputFile}
+  echo -e "${Gre}The file ${Blu}${tempOutputFile} ${Gre}generated in${RCol}: ${SECONDS}s" &&
+  babel "${jsonParser}" | node              # only parse JSON if we have a source to parse it from
 # if the path exists but is empty
 elif [ -d "${yourpathSanitized}" ] && [ ! "$(ls $yourpathSanitized)" ]; then
-  echo -e "${Whi}[ERROR 002]: ${Yel}The path to the local repositories ${Red}'${yourpath}'${Yel}, which is set on the file ${Blu}'gitlogg-generate-log.sh' ${UYel}exists, but is empty!${RCol}"
+  echo -e "${Whi}[ERROR 002]: ${Yel}The path to the local repositories ${Red}'${yourpath}'${Yel}, which is set on the file ${Blu}'${thisFile}' ${UYel}exists, but is empty!${RCol}"
   echo -e "${Yel}Please move the repos to ${Red}'${yourpath}'${Yel} or update the variable ${Pur}'yourpath'${Yel} to reflect the absolute path to the directory where the repos are located.${RCol}"
 # if the path does not exists
 elif [ ! -d "${yourpathSanitized}" ]; then
-  echo -e "${Whi}[ERROR 001]: ${Yel}The path to the local repositories ${Red}'${yourpath}'${Yel}, which is set on the file ${Blu}'gitlogg-generate-log.sh' ${UYel}does not exist!${RCol}"
+  echo -e "${Whi}[ERROR 001]: ${Yel}The path to the local repositories ${Red}'${yourpath}'${Yel}, which is set on the file ${Blu}'${thisFile}' ${UYel}does not exist!${RCol}"
   echo -e "${Yel}Please create ${Red}'${yourpath}'${Yel} and move the repos under it, or update the variable ${Pur}'yourpath'${Yel} to reflect the absolute path to the directory where the repos are located.${RCol}"
 fi

gitlogg/gitlogg-generate-log.sh renamed to scripts/gitlogg-generate-log.sh

@@ -1,7 +1,8 @@
 var fs = require('fs'),
     path = require('path'),
     chalk = require('chalk'),
-    output_file = './gitlogg.json';
+    output_file_temp = '_tmp/gitlogg.tmp',
+    output_file = '_output/gitlogg.json';
 
 console.log(chalk.yellow('Generating JSON output...'));
 
@@ -16,7 +17,7 @@ var changes = function(data, index) {
 
 console.time(chalk.green('JSON output generated in'));
 
-var output = fs.readFileSync('gitlogg.tmp', 'utf8')
+var output = fs.readFileSync(output_file_temp, 'utf8')
   .trim()
   .split('\n')
   .map(line => line.split('\\t'))

gitlogg/gitlogg-parse-json.js renamed to scripts/gitlogg-parse-json.js

@@ -1,3 +1,6 @@
 #!/bin/bash
 
+my_dir="$(dirname "$0")"
+cd $my_dir
+
 bash gitlogg-generate-log.sh $@