Author: Pannous

combine.py

@@ -5,6 +5,8 @@
 from PIL import Image # Image.open(test_image)
 # import cv2 # cv2.imwrite(output_path, img)
 # import skimage #skimage.io.imread
+# server='0.0.0.0'
+server='87.118.88.144' #dev03'
 
 class Box(object):
     def __init__(self, **kwargs):
@@ -14,21 +16,14 @@ def __init__(self, **kwargs):
 image_file='test_image.png'
 test_file='test_out.png'
 image = Image.open(image_file)
-
 with open(image_file, 'rb') as f:
-    r = requests.post('http://0.0.0.0:8769/?json=1', files={'image': f})
-    print(r.text)
-    ocr=json.loads(r.text.replace(""",'"'))
-    for line in ocr['text_lines']:
+    r = requests.post('http://'+server+':8769/?json=1', files={'image': f})
+    raw=r.text.replace(""",'"')
+    print(raw)
+    boxes=json.loads(raw)
+    for line in boxes['text_lines']:
       print(line)
       b=Box(**line)
       print(b.x0)
       word=image.crop((b.x0-5, b.y0-5, b.x2+15, b.y2+15))
       word.save(open(test_file, 'wb'))
-
-# h = httplib.HTTPConnection('http://0.0.0.0:8769')
-# headers={}
-# data=[]
-# h.request('POST', '/?json=1', data, headers)
-# r = h.getresponse()
-# result=r.read()  

mouse_prediction.py

@@ -1,13 +1,11 @@
 #!/usr/bin/env python
+import numpy
+
+import numpy as np
 import sys
 
 import matplotlib.pyplot as plt
-import numpy
-import numpy as np
-# import gtk
-# gtk.set_interactive(False)
 import pyscreenshot
-import tensorflow as tf
 
 from text_recognizer import predict_tensor
 
@@ -16,35 +14,27 @@
 except Exception as ex:
   import tkinter
 
+if sys.platform == 'Windows':
+  import win32api # GetCursorPos
+
 app = tkinter.Tk()  # must be declared before Mat
 
-# import cv2
 plt.matshow([[1, 0], [0, 1]], fignum=1)
-# print(dir(plt))
-# help(plt)
-# ax.patch.set_facecolor('None') or ax.patch.set_visible(False).
 plt.draw()
 
 # if mac
 # system('''/usr/bin/osascript -e 'tell app "Finder" to set frontmost of process "Python" to true' ''')
 
 i = 0
-width = 256
-height = 256
-
 
 def get_mouse_position():
   if sys.platform == 'Windows':
-    import win32api
     x, y = win32api.GetCursorPos()
   else:
     x, y = app.winfo_pointerxy()
   return x, y
 
 
-get_mouse_position()
-session = tf.Session()
-
 if __name__ == "__main__":
   while 1:
     x, y = get_mouse_position()
@@ -60,15 +50,19 @@ def get_mouse_position():
     # image = pyscreenshot.grab([x, y, x + w, y + h])
     mat = np.array(image) / 255.0  # RGBA: h*w*4
 
-
     lines=numpy.average(mat, axis=1)
+    # todo make model robust to extra text
     argmax = numpy.argmax(lines) # most white
     argmin = numpy.argmin(lines) # most black
     if(argmax<argmin):
       mat[:,:argmax,:]=1. # fill white above
+    if(argmin<argmax):
+      mat[:,argmax:,:]=1. # fill white below
+    # todo: what if invert image!?
 
     tensor = mat
     print(tensor.shape)
+    # tensor=cv2.resize(tensor,(64,512))
     if len(tensor.shape) == 2:
       tensor = tensor.transpose((1, 0))
       tensor = tensor[np.newaxis, :, :, np.newaxis]
@@ -77,27 +71,19 @@ def get_mouse_position():
       tensor = tensor.transpose((2, 1, 0))  # 4*w*h
       tensor = tensor[:, :, :, np.newaxis]
 
-    # tensor=cv2.resize(tensor,(64,512))
+    # mat = 1 - 2 * mat / 255.  # norm [-1,1] !
+    # mat = 1 - mat / 255.  # norm [0,1]! black=1
+    # mat = mat / 255.  # norm [0,1]! black=0 (default)
+
     """
 
  TEST Text 01234 Hello     <- point your mouse here
  
 """
-    # help(image.show) Displays this image via preview. This method is mainly intended for debugging purposes
-    # array = numpy.array(image.getdata())  # (1, 4000, 4)
-    #
-    # mat = array.reshape(image.height, image.width, -1)[:, :, 0]
-    # if size> image.height:
-    # 	mat=numpy.pad(mat, (0,  size- image.height), 'constant', constant_values=255) # 1==white!
-
-    # mat = 1 - 2 * mat / 255.  # norm [-1,1] !
-    # mat = 1 - mat / 255.  # norm [0,1]! black=1
-    # mat = mat / 255.  # norm [0,1]! black=0 (default)
 
     plt.matshow(mat, fignum=1)
     # plt.imshow(image)
 
-
     histogram = numpy.histogram(mat, bins=10, range=None, normed=False, weights=None, density=None)
     print(argmax)
 

text_recognizer.py

@@ -10,58 +10,24 @@
 from keras.layers.convolutional import Conv2D, MaxPooling2D
 from keras.layers.merge import add, concatenate
 from keras.layers.recurrent import GRU
-from keras.models import Model
+from keras.models import Model, load_model 
 
 # weight_file = 'best_weights.h5'
-weight_file = 'current_weights.h5'
+# weight_file = 'current_weights.h5'
+weight_file = 'weights_ascii.h5'
 
-# alphabet = u'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 '
-alphabet = u'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZäöüÄÖÜß0123456789!@#$%^&*()[]{}-_=+\\|"\'`;:/.,?><~ '
+
+# chars = u'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 '
+chars = u'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZäöüÄÖÜß0123456789!@#$%^&*()[]{}-_=+\\|"\'`;:/.,?><~ '
 
 
 global model
 model=None
 
 def load_model():
   global model
-  # Model similar to image_ocr.py
-  rnn_size = 1024
-  dropout = 0.2
-  pool_size = 2
-  kernel_size = (3, 3)
-  time_dense_size = 32
-  conv_filters = 16
-  img_h = 64
-  img_w = 512
-
-  act = 'relu'
-  input_data = Input(name='the_input', shape=(img_w, img_h, 1), dtype='float32')
-  inner = Conv2D(conv_filters, kernel_size, padding='same', activation=act, name='conv1')(input_data)
-  inner = MaxPooling2D(pool_size=(pool_size, pool_size), name='max1')(inner)
-  inner = Conv2D(conv_filters, kernel_size, padding='same', activation=act, name='conv2')(inner)
-  inner = MaxPooling2D(pool_size=(pool_size, pool_size), name='max2')(inner)
-
-  conv_to_rnn_dims = (img_w // (pool_size ** 2), (img_h // (pool_size ** 2)) * conv_filters)
-  inner = Reshape(target_shape=conv_to_rnn_dims, name='reshape')(inner)
-
-  inner = Dropout(rate=dropout, name='dropout_dense1a')(inner)
-  inner = Dense(time_dense_size, activation=act, name='dense1')(inner)
-  inner = Dropout(rate=dropout, name='dropout_dense1b')(inner)
-
-  # Two layers of bidirectional GRUs
-  gru_1 = GRU(rnn_size, return_sequences=True, dropout=0.3, name='gru1')(inner)
-  gru_1b = GRU(rnn_size, return_sequences=True, go_backwards=True, name='gru1_b')(inner)
-  gru1_merged = add([gru_1, gru_1b])
-  gru_2 = GRU(rnn_size, return_sequences=True, dropout=0.3, name='gru2')(gru1_merged)
-  gru_2b = GRU(rnn_size, return_sequences=True, go_backwards=True, name='gru2_b')(gru1_merged)
-
-  dense2 = Dense(len(alphabet) + 1, name='dense2')
-  inner = dense2(concatenate([gru_2, gru_2b]))
-  y_pred = Activation('softmax', name='softmax')(inner)
-  model = Model(inputs=input_data, outputs=y_pred)
-  model.summary()
-
-  model.load_weights(weight_file, reshape=True, by_name=True)
+  model = load_model(weight_file)
+  # model.load_weights(weight_file, reshape=True, by_name=True)
 
 def predict_tensor(images):
   if not model: load_model()
@@ -74,10 +40,10 @@ def decode_labels(labels):
   ret = []
   for c in labels:
     # ret += chr(c)
-    if c == len(alphabet):
+    if c == len(chars):
       ret.append("")
     else:
-      ret.append(alphabet[c])
+      ret.append(chars[c])
   return "".join(ret)