Merge pull request #2199 from magnumripper/rexgen-resume

regex 1.4 integration, and full restore logic

.pre-commit.sh

@@ -36,7 +36,7 @@ fi
 ########################################################
 # Tests 2/3/4/5. Make sure source does not have execute bit set
 ########################################################
-TXT_FILE_PATTERN='\.(cl|c|h|txt|S|s|in|chr|conf|txt|md|asm|cin|dep|gost|legacy|macros|stub|m4|cu|bash_completion|zsh_completion|lst)(\..+)?$'
+TXT_FILE_PATTERN='\.(ac|cl|c|h|txt|S|s|in|chr|conf|txt|md|asm|cin|dep|gost|legacy|macros|stub|m4|cu|bash_completion|zsh_completion|lst)(\..+)?$'
 TXT_FILES=`git diff --cached --name-only | grep -E $TXT_FILE_PATTERN`
 TXT_FILES="$TXT_FILES"`git diff --cached --name-only | grep -E ^doc/`
 if [ "x$TXT_FILES" != "x" ] ; then

doc/README.librexgen

@@ -14,8 +14,8 @@ The current options we have are:
               will be run to convert the reg-ex AFTER the word has been
               prepared and delivered.  Fun things like f mapping to ph
               or M mapping to |\/| can be done.  Case can also easily be
-              done here, but the case option is also used. These options
-              are stored in teh regex_alphabest.conf file in ./run dir
+              done here if the case option is also used. These options
+              are stored in the regex_alphabets.conf file in ./run dir
               of JtR.  There are these current alphabets:
               The default (if just =alpha is used). It is an empty
               alphabet, nothing will change.  alpha:case  This is the
@@ -31,7 +31,9 @@ The current options we have are:
               letter replacements which likely are not seen in garden
               variety passwords. h -> h H  |-|  ]-[  }-{  (-)  )-(  }{  #
               is one example of alpha:leet3.
-
+              alpha:ascii2nonascii is a alphabet which will convert ascii
+              characters into non ascii utf8 characters which 'look'
+              similar (i.e. a with grave, umlat, accent, hook, etc)
 
 
 Currently, rexgen can be used stand alone, OR with wordlist and rules.
@@ -108,11 +110,14 @@ rexgen '[0-2]password[A-C]'
 1passwordC
 2passwordC
 
-Once you are at this point, rexgen is installed.  JtR Makefile will
-(should) autodetect the library, and now build in code to use it.
+Once you are at this point, rexgen is installed.  JtR ./configure should
+autodetect the library, and now build in code to use it. NOTE, JtR requires
+at least version 1.4.x to work properly. There were some bug fixes done
+when save/restore was added to rexgen, and versions of rexgen prior to
+this, simply do not work properly.
 
 Note, JtR help document is NOT here to troubleshoot installation of this
-library.  A simple how to do it (when thing work write), is what was listed.
+library.  A simple how to do it (when thing work right), is what was listed.
 If there problems beyond what is addressed in this document, then the lib's
 author is probably the best way to go.  A quick 'help me' post on john-users
 may get the proper answer, AND if we get a quick resolve, we will likely

src/configure

@@ -10202,18 +10202,23 @@ $as_echo "$ac_cv_search_rexgen_version_int" >&6; }
 ac_res=$ac_cv_search_rexgen_version_int
 if test "$ac_res" != no; then :
   test "$ac_res" = "none required" || LIBS="$ac_res $LIBS"
-  if test "$cross_compiling" = yes; then :
+  { $as_echo "$as_me:${as_lineno-$LINENO}: checking librexgen minimum version" >&5
+$as_echo_n "checking librexgen minimum version... " >&6; } && if test "$cross_compiling" = yes; then :
   { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
 as_fn_error $? "cannot run test program while cross compiling
 See \`config.log' for more details" "$LINENO" 5; }
 else
   cat confdefs.h - <<_ACEOF >conftest.$ac_ext
 /* end confdefs.h.  */
-int main() { return ! (rexgen_version_int() >= 0x010300); }
+int main() { return ! (rexgen_version_int() >= 0x010400); }
 _ACEOF
 if ac_fn_c_try_run "$LINENO"; then :
-  using_rexgen=yes
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: usable" >&5
+$as_echo "usable" >&6; } && using_rexgen=yes
+else
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: TOO old. Rexgen not usable!" >&5
+$as_echo "TOO old. Rexgen not usable!" >&6; }
 fi
 rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \
   conftest.$ac_objext conftest.beam conftest.$ac_ext

src/configure.ac

@@ -507,8 +507,9 @@ using_rexgen=no
 if test "x$enable_rexgen" != xno ; then
   AC_CHECK_HEADER([librexgen/version.h],
     [AC_SEARCH_LIBS([rexgen_version_int],[rexgen librexgen],
-      [AC_TRY_RUN([int main() { return ! (rexgen_version_int() >= 0x010300); }],
-        [using_rexgen=yes])]
+      [AC_MSG_CHECKING([librexgen minimum version])] && [AC_TRY_RUN([int main() { return ! (rexgen_version_int() >= 0x010400); }],
+        AC_MSG_RESULT([usable]) && [using_rexgen=yes],
+		AC_MSG_RESULT([TOO old. Rexgen not usable!]))]
     )]
   )
   if test "x$using_rexgen" == "xyes" ; then

src/cracker.c

@@ -52,6 +52,7 @@
 #include "mask_ext.h"
 #include "mask.h"
 #include "unicode.h"
+#include "cracker.h"
 #include "john.h"
 #include "fake_salts.h"
 #include "sha.h"
@@ -77,6 +78,7 @@
 static clock_t salt_time = 0;
 #endif
 
+static fix_state_fp fp_fix_state;
 static struct db_main *crk_db;
 static struct fmt_params crk_params;
 static struct fmt_methods crk_methods;
@@ -758,6 +760,11 @@ static int crk_process_event(void)
 	return event_abort;
 }
 
+void crk_set_hybrid_fix_state_func_ptr(fix_state_fp fp)
+{
+	fp_fix_state = fp;
+}
+
 static int crk_password_loop(struct db_salt *salt)
 {
 	void ext_hybrid_fix_state(void);
@@ -776,7 +783,8 @@ static int crk_password_loop(struct db_salt *salt)
 	if (event_pending && crk_process_event())
 		return -1;
 
-	ext_hybrid_fix_state();
+	if (fp_fix_state)
+		fp_fix_state();
 
 	count = crk_key_index;
 	match = crk_methods.crypt_all(&count, salt);

src/cracker.h

@@ -60,4 +60,11 @@ extern int crk_reload_pot(void);
  * Exported for stacked modes
  */
 extern void (*crk_fix_state)(void);
+
+/*
+ * This needs set for 2nd level save/resume code to get proper
+ * information stashed away so resume is done properly.
+ */
+typedef void (*fix_state_fp)();
+extern void crk_set_hybrid_fix_state_func_ptr(fix_state_fp fp);
 #endif

src/external.c

@@ -645,6 +645,7 @@ int do_external_hybrid_crack(struct db_main *db, const char *base_word) {
 	if (first) {
 		strcpy(int_hybrid_base_word, base_word);
 		rec_init_hybrid(save_state_hybrid);
+		crk_set_hybrid_fix_state_func_ptr(ext_hybrid_fix_state);
 		first = 0;
 		just_restored = rec_restored;
 		if (rec_restored) {

src/recovery.c

@@ -55,6 +55,7 @@
 #include "status.h"
 #include "recovery.h"
 #include "external.h"
+#include "regex.h"
 #include "john.h"
 #include "mask.h"
 #include "unicode.h"
@@ -652,12 +653,12 @@ void rec_restore_mode(int (*restore_mode)(FILE *file))
 			if (ext_restore_state_hybrid(buf, rec_file))
 				rec_format_error("external-hybrid");
 		}
-		/*
+#if HAVE_REXGEN
 		else if (!strncmp(buf, "rex-v", 5)) {
 			if (rexgen_restore_state_hybrid(buf, rec_file))
 				rec_format_error("rexgen-hybrid");
 		}
-		*/
+#endif
 		if (!strcmp(buf, "slt-v1")) {
 			restore_salt_state();
 		}

src/regex.c

@@ -35,7 +35,20 @@
 #define UNICODE
 #define _UNICODE
 
+#ifdef _MSC_VER
+char *stpcpy(char *dst, const char *src) {
+	strcpy(dst, src);
+	return dst + strlen(dst);
+}
+#endif
+
 char *rexgen_alphabets[256];
+static c_iterator_ptr iter = NULL;
+static c_regex_ptr regex_ptr = NULL;
+static char *save_str;
+static const char *cur_regex, *save_regex;
+static char *restore_str, *restore_regex;
+static int save_str_len;
 
 static void fix_state(void)
 {
@@ -50,6 +63,52 @@ static void save_state(FILE *file)
 {
 }
 
+int rexgen_restore_state_hybrid(const char *sig, FILE *file)
+{
+	if (!strncmp(sig, "rex-v1", 6))
+	{
+		int len, ret;
+		ret = fscanf(file, "%d\n", &len);
+		if (ret != 1) return 1;
+		restore_regex = mem_alloc_tiny(len+2, 8);
+		fgetl(restore_regex, len+1, file);
+		ret = fscanf(file, "%d\n", &len);
+		if (ret != 1) return 1;
+		restore_str = mem_alloc_tiny(len+2, 8);
+		fgetl(restore_str, len+1, file);
+		log_event("resuming a regex expr or %s and state of %s\n", restore_regex, restore_str);
+		return 0;
+	}
+	return 1;
+}
+
+static void save_state_hybrid(FILE *file)
+{
+	if (save_str && strlen(save_str)) {
+		fprintf(file, "rex-v1\n");
+		fprintf(file, "%d\n", (int)strlen(save_regex));
+		fprintf(file, "%s\n", save_regex);
+		fprintf(file, "%d\n", (int)strlen(save_str));
+		fprintf(file, "%s\n", save_str);
+	}
+}
+
+static void rex_hybrid_fix_state()
+{
+	char *dstptr=0;
+	if (iter)
+		c_iterator_get_state(iter, &dstptr);
+	if (dstptr) {
+		if (strlen(dstptr) > save_str_len) {
+			save_str_len = strlen(dstptr)+256;
+			MEM_FREE(save_str);
+			save_str = mem_alloc(save_str_len+1);
+		}
+		strcpy(save_str, dstptr);
+		save_regex = cur_regex;
+	}
+}
+
 static int restore_state(FILE *file)
 {
 	return 0;
@@ -76,6 +135,7 @@ static void rexgen_setlocale()
 	}
 }
 
+// Would be nice to have SOME way to be thread safe!!!
 static char BaseWord[1024];
 
 size_t callback(char* dst, const size_t buffer_size)
@@ -134,24 +194,39 @@ int do_regex_hybrid_crack(struct db_main *db, const char *regex,
                           const char *base_word, int regex_case, const char *regex_alpha)
 {
 	c_simplestring_ptr buffer = c_simplestring_new();
-	c_iterator_ptr iter = NULL;
-	c_regex_ptr regex_ptr = NULL;
 	char word[PLAINTEXT_BUFFER_SIZE];
 	static int bFirst = 1;
 	static int bALPHA = 0;
 	int max_len = db->format->params.plaintext_length;
 	int retval;
 
+	cur_regex = regex;
 	if (options.req_maxlength)
 		max_len = options.req_maxlength;
 
+	strcpy(BaseWord, base_word);
 	if (bFirst) {
 		bFirst = 0;
 		rexgen_setlocale();
 		if (regex_alpha && !strncmp(regex_alpha, "alpha", 5)) {
 			bALPHA = 1;
 			SetupAlpha(regex_alpha);
 		}
+		rec_init_hybrid(save_state_hybrid);
+		crk_set_hybrid_fix_state_func_ptr(rex_hybrid_fix_state);
+
+		regex_ptr = c_regex_cb(regex, callback);
+		if (!regex_ptr) {
+			c_simplestring_delete(buffer);
+			fprintf(stderr,
+				"Error, invalid regex expression.  John exiting now  base_word=%s  Regex= %s\n",
+				base_word, regex);
+			error();
+		}
+		iter = c_regex_iterator(regex_ptr);
+
+		if (restore_str)
+			c_iterator_set_state(iter, restore_str);
 	}
 
 	if (bALPHA) {
@@ -190,7 +265,6 @@ int do_regex_hybrid_crack(struct db_main *db, const char *regex,
 		}
 	}
 
-	strcpy(BaseWord, base_word);
 	if (!regex[0]) {
 		if (options.mask) {
 			if (do_mask_crack(fmt_null_key)) {
@@ -208,15 +282,6 @@ int do_regex_hybrid_crack(struct db_main *db, const char *regex,
 		goto out;
 	}
 
-	regex_ptr = c_regex_cb(regex, callback);
-	if (!regex_ptr) {
-		c_simplestring_delete(buffer);
-		fprintf(stderr,
-		        "Error, invalid regex expression.  John exiting now  base_word=%s  Regex= %s\n",
-		        base_word, regex);
-		error();
-	}
-	iter = c_regex_iterator(regex_ptr);
 	while (c_iterator_next(iter)) {
 		c_iterator_value(iter, buffer);
 		c_simplestring_to_utf8_string(buffer, &word[0], sizeof(word));
@@ -240,38 +305,37 @@ int do_regex_hybrid_crack(struct db_main *db, const char *regex,
 
 out:
 	c_simplestring_delete(buffer);
-	c_regex_delete(regex_ptr);
-	c_iterator_delete(iter);
 	return retval;
 }
 
 void do_regex_crack(struct db_main *db, const char *regex)
 {
 	c_simplestring_ptr buffer = c_simplestring_new();
-	c_iterator_ptr iter = NULL;
-	c_regex_ptr regex_ptr = NULL;
 	char word[PLAINTEXT_BUFFER_SIZE];
 	int max_len = db->format->params.plaintext_length;
 
 	if (options.req_maxlength)
 		max_len = options.req_maxlength;
 
-	if (john_main_process)
-		fprintf(stderr, "Warning: regex mode currently can't be "
-		        "resumed if aborted\n");
-
+	cur_regex = regex;
 	rexgen_setlocale();
 	status_init(&get_progress, 0);
 	rec_restore_mode(restore_state);
 	rec_init(db, save_state);
 	crk_init(db, fix_state, NULL);
+	rec_init_hybrid(save_state_hybrid);
+
 	regex_ptr = c_regex_cb(regex, callback);
 	if (!regex_ptr) {
 		fprintf(stderr,
 		        "Error, invalid regex expression.  John exiting now\n");
 		error();
 	}
 	iter = c_regex_iterator(regex_ptr);
+	if (restore_str) {
+		c_iterator_set_state(iter, restore_str);
+		restore_str = 0;
+	}
 	while (c_iterator_next(iter)) {
 		c_iterator_value(iter, buffer);
 		c_simplestring_to_utf8_string(buffer, &word[0], sizeof(word));

src/regex.h

@@ -52,6 +52,8 @@ void do_regex_crack(struct db_main *db, const char *regex);
 int do_regex_hybrid_crack(struct db_main *db, const char *regex,
                           const char *base_word, int bCase, const char *regex_alpha);
 char *prepare_regex(char *regex, int *bCase, char **regex_alpha);
+int rexgen_restore_state_hybrid(const char *sig, FILE *file);
+
 #else
 #undef HAVE_REXGEN
 #define do_regex_hybrid_crack(a,word,b,c) crk_process_key(word)