use sedlex for utf-8 aware lexing (closes #5163)

.merlin

@@ -2,4 +2,5 @@ S src/**
 B _build/src/**
 S libs/**
 B libs/**
-B +threads
+B +threads
+PKG sedlex

.travis.yml

@@ -33,6 +33,9 @@ install_linux: &install_linux
       ocaml-native-compilers
       ocaml-findlib
       camlp4
+  - wget https://raw.github.com/ocaml/opam/master/shell/opam_installer.sh -O - | sh -s /usr/local/bin system
+  - export OPAMYES=1
+  - opam install sedlex
   # Setup database
   - travis_retry sudo apt-get install mysql-server-5.6 -y
   - mysql -u root -e "create user travis@localhost identified by '';"
@@ -43,7 +46,7 @@ install_linux: &install_linux
   - java -version
   # Build haxe
   - make package_src -s
-  - make -s
+  - opam config exec -- make -s
   - make package_bin -s
   - ls -l out
   - export PATH="$PATH:$TRAVIS_BUILD_DIR"
@@ -56,7 +59,7 @@ install_osx: &install_osx
   - travis_retry brew install opam;
   - export OPAMYES=1
   - opam init
-  - opam install camlp4 ocamlfind
+  - opam install camlp4 sedlex ocamlfind
   - eval `opam config env`
   # Install neko
   - travis_retry brew install neko --HEAD;

Makefile

@@ -8,7 +8,7 @@
 #  - use 'make -f Makefile.win' to build for Windows
 #  - use 'make MSVC=1 -f Makefile.win' to build for Windows with OCaml/MSVC
 #
-.SUFFIXES : .ml .mli .cmo .cmi .cmx .mll .mly
+.SUFFIXES : .ml .mli .cmo .cmi .cmx .mly
 
 INSTALL_DIR=$(DESTDIR)/usr
 INSTALL_BIN_DIR=$(INSTALL_DIR)/bin
@@ -29,7 +29,7 @@ STATICLINK?=0
 
 HAXE_DIRECTORIES=compiler context generators generators/gencommon macro filters optimization syntax typing display
 EXTLIB_LIBS=extlib extc neko javalib ziplib swflib xml-light ttflib ilib objsize pcre
-FINDLIB_LIBS=unix str threads
+FINDLIB_LIBS=unix str threads sedlex
 
 # Includes, packages and compiler
 

appveyor.yml

@@ -34,7 +34,7 @@ install:
     - 7z x "opam32.tar.xz" -so | 7z x -aoa -si -ttar
     - '%CYG_ROOT%/bin/bash -lc "cd \"$OLDPWD\" && bash opam32/install.sh"'
     - '%CYG_ROOT%/bin/bash -lc "opam init mingw \"https://github.com/fdopen/opam-repository-mingw.git\" --comp 4.02.3+mingw32c --switch 4.02.3+mingw32c --auto-setup --yes"'
-    - '%CYG_ROOT%/bin/bash -lc "opam install camlp4 ocamlfind --yes"'
+    - '%CYG_ROOT%/bin/bash -lc "opam install camlp4 sedlex ocamlfind --yes"'
     # Install neko
     - choco install neko --prerelease --ignore-dependencies -s 'https://ci.appveyor.com/nuget/neko' -y
     - choco install chocolatey-core.extension php --ignore-dependencies -y

compile.sh

@@ -3,10 +3,6 @@ if [ ! -f "_build/src/syntax/parser.ml" -o "src/syntax/parser.mly" -nt "_build/s
 	camlp4o -impl src/syntax/parser.mly -o _build/src/syntax/parser.ml
 fi
 
-if [ ! -f "_build/src/syntax/lexer.ml" -o "src/syntax/lexer.mll" -nt "_build/src/syntax/lexer.ml" ]; then
-	ocamllex -o _build/src/syntax/lexer.ml src/syntax/lexer.mll
-fi
-
 if [ "$1" -eq "0" -a ! -f "_build/src/compiler/version.ml" ]; then
 	echo let version_extra = None > _build/src/compiler/version.ml
 fi