semantic-matlab.el

;;; semantic-matlab.el --- Semantic details for MATLAB files

;;; Copyright (C) 2004, 2005, 2008, 2012, 2013 Eric M. Ludlam: The Mathworks, Inc

;; Author: Eric M. Ludlam <eludlam@mathworks.com>
;; X-RCS: $Id$

;; This file is not part of GNU Emacs.

;; This is free software; you can redistribute it and/or modify
;; it under the terms of the GNU General Public License as published by
;; the Free Software Foundation; either version 2, or (at your option)
;; any later version.

;; This software is distributed in the hope that it will be useful,
;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;; GNU General Public License for more details.

;; You should have received a copy of the GNU General Public License
;; along with GNU Emacs; see the file COPYING.  If not, write to the
;; Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
;; Boston, MA 02110-1301, USA.

;;; Commentary:
;;
;; Parse a MATLAB M file for use w/ CEDET/Semantic
;;
;; The MATLAB language is pretty simple from a functional standpoint in that
;; you can only declare functions.  In addition, the language itself is not
;; expressable in a yacc style grammar.  It is therefore more expedient
;; to scan for regular expressions.
;;
;; Caveat: MCOS classes have property declarations. @todo - support them

(require 'mode-local)
(require 'semantic)
(eval-and-compile
  (condition-case nil
      (require 'semantic-format)
    (error (require 'semantic/format)))
  (condition-case nil
      (require 'semantic-dep)
    (error (require 'semantic/dep))))
(require 'matlab)
(require 'semanticdb-matlab)

;;; Code:
(defvar semantic-matlab-system-paths-include '("toolbox/matlab/funfun" "toolbox/matlab/general")
  "List of include paths under `semantic-matlab-root-directory'.
These paths will be parsed recursively by semantic.  Class and
private directories will be omitted here.")

(defvar semantic-matlab-root-directory
  (let* ((mlab (locate-file "matlab" exec-path))
         (mlint (and (boundp 'mlint-program)
                     mlint-program))
         (exe (or mlab mlint)))
    (if exe
        (let ((dir
               (or (file-symlink-p exe)
                   exe)))
          ;; If we have a dir, take everything until /bin as root dir.
          (string-match "\\(.*\\)/bin.*" dir)
          (match-string 1 dir))
      (message "semantic-matlab: Could not find MATLAB executable in path.")
      nil))
  "Root directory of MATLAB installation.
Will be automatically determined by MATLAB or mlint executable.
Use `semantic-matlab-system-paths-include' to let semantic know
which system directories you would like to include when doing
completions.")

(defun semantic-matlab-root-directory ()
  "Calculate the current MATLAB root directory."
  (if (matlab-shell-active-p)
      (matlab-shell-matlabroot)
    semantic-matlab-root-directory))

;; The version of this variable in MATLAB.el is not condusive to extracting
;; the information we need.
(defvar semantic-matlab-match-function-re
  "\\(^\\s-*function\\b[ \t\n.]*\\)\\(\\[[^]]+\\]\\s-*=\\|\\w+\\s-*=\\|\\)\\s-*\\(\\(\\sw\\|\\s_\\)+\\)\\>"
  "Expression to match a function start line.")

;; This function may someday be a part of matlab.el.
;; It does the raw scan and split for function tags.
(defun semantic-matlab-function-tags (&optional buffer)
  "Find all MATLAB function tags in BUFFER.
Return argument is:
  (START END RETURNVARS NAME ARGUMENTS DOCSTRING).
Note that builtin functions from MATLAB will always return
START=END=0 and no arguments or return values."
  (save-excursion
    (if buffer (set-buffer buffer))
    (let ((re semantic-matlab-match-function-re)
          start ret fn arg end doc
          (taglist nil))
      (goto-char (point-min))
      (if (and (string-match (format "^%s" (semantic-matlab-root-directory))
                             (buffer-file-name))
               (looking-at "%\\([A-Z0-9_]+\\)\\s-+\\(.*\\)\\s-*$"))
          ;; This is a builtin function, ie there's no function line.
          ;; Hence we must use function name from the doc string.
          ;; FIXME
          ;; How can we get function arguments/return vals for builtin func's?
          (setq taglist
                (cons (list 0 0 nil (downcase (match-string-no-properties 1))
                            nil (match-string-no-properties 2) t)
                      taglist))
        ;; this is a either not builtin or a user function
        (while (re-search-forward re nil t)
          (setq start (match-beginning 0)
                ret (buffer-substring-no-properties
                     (match-beginning 2) (match-end 2))
                fn (buffer-substring-no-properties
                    (match-beginning 3) (match-end 3))
                arg (buffer-substring-no-properties
                     (match-end 3) (save-excursion
                                     (matlab-end-of-command)
                                     (point)))
                doc (save-excursion
                      (forward-line)
                      (beginning-of-line)
                      ;; snarf doc string
                      (cond
                        ;; Mathworks standard
                        ((looking-at "%[A-Z0-9_]+\\s-+\\(.*\\)\\s-*$")
                         (match-string-no-properties 1))
                        ;; lookfor string
                        ((looking-at "%\\s-+\\(.*\\)\\s-*$")
                         (match-string-no-properties 1))
                        ;; otherwise simply snarf first line of
                        ;; comments under function declaration
                        (t
                         (re-search-forward "[^[:blank:][:cntrl:]]" nil t)
                         (backward-char)
                         (if (looking-at "%\\s-+\\(.*\\)")
                             (match-string-no-properties 1)
                           nil))))
                end (save-excursion
                      (goto-char start)
                      (if matlab-functions-have-end
                          (condition-case nil
                              ;; If we get a failure, we should at least
                              ;; return whatever we got so far.
                              (matlab-forward-sexp)
                            (error (point-max)))
                        (matlab-end-of-defun))
                      (point)))
          (setq taglist
                (cons (list start end
                            (split-string ret "[][,=. \t\n]+" t)
                            fn
                            (split-string arg "[(), \n\t.]+" t)
                            doc
                            nil)
                      taglist))))
      (nreverse taglist))))

(defun semantic-matlab-parse-oldstyle-class (tags &optional buffer)
  "Check if BUFFER with current TAGS is the constructor of a class.
If this is the case, retrieve attributes from the buffer and scan
the whole directory for methods.  The function returns a single tag
describing the class.  This means that in semantic-matlab, the
old-style MATLAB classes are linked to the constructor file."
  (let* ((name (buffer-file-name buffer))
         class method methods retval attributes)
    (when (string-match ".*/@\\(.*?\\)/\\(.*?\\)\\.m" name)
      ;; this buffer is part of a class - check
      (setq class (match-string 1 name))
      (setq method (match-string 2 name))
      (when (string= class method)      ; is this the constructor?
        ;; get attributes of the class
        ;; TODO - we blindly assume the constructor is correctly defined
        (setq retval (semantic-tag-get-attribute (car tags) :return))
        (goto-char (point-min))
        ;; search for attributes
        (while (re-search-forward
                (concat "^\\s-*" (car retval)
                        "\\.\\([A-Za-z0-9_]+\\)\\s-*=\\s-*\\(.+\\);")
                nil t)
          (push (list (match-string-no-properties 1) ; name
                      (match-string-no-properties 2)) ; default value
                attributes))
        ;; now scan the methods
        (dolist (cur (delete class
                             (nthcdr 2
                                     (assoc class semanticdb-matlab-user-class-cache))))
          (push
           (semantic-tag-put-attribute
            (car (semanticdb-file-stream
                  (concat
                   (file-name-directory name)
                   cur ".m")))
            :typemodifiers '("public"))
           methods))
        ;; generate tag
        (semantic-tag-new-type
         class
         "class"
         (append
          (mapcar (lambda (cur)
                    (semantic-tag-new-variable
                     (car cur) nil (cdr cur)
                     :typemodifiers '("public")))
                  attributes)
          methods)
         nil
         :typemodifiers '("public"))))))

(defun semantic-matlab-find-oldstyle-classes (files)
  "Scan FILES for old-style Matlab class system.
Returns an alist with elements (CLASSNAME LOCATION METHODS)."
  (let (classes temp tags)
    (dolist (cur files)
      ;; scan file path for @-directory
      (when (string-match "\\(.*\\)/@\\(.*?\\)/\\(.*?\\)\\.m" cur)
        (if (setq temp
                  (assoc (match-string 2 cur) classes))
            (nconc temp `(,(match-string 3 cur)))
          (push `(,(match-string 2 cur) ,(match-string 1 cur)
                   ,(match-string 3 cur)) classes))))
    classes))

;;; BEGIN PARSER
;;
(defun semantic-matlab-parse-region (&rest ignore)
  "Parse the current MATLAB buffer for function definitions.
IGNORE any arguments which specify a subregion to parse.
Each tag returned is a semantic FUNCTION tag.  See
`semantic-tag-new-function'."
  (semanticdb-matlab-cache-files)
  (let ((raw (condition-case nil
                 ;; Errors from here ought not to be propagated.
                 (semantic-matlab-parse-functions)
               (error nil)))
        tags ctags)
    (setq tags (mapcar 'semantic-matlab-expand-tag raw))
    ;; check if this is a class constructor
    (setq ctags (list (semantic-matlab-parse-oldstyle-class tags)))
    (if (car ctags)
        ctags
      tags)))

(defun semantic-matlab-parse-changes ()
  "Parse all changes for the current MATLAB buffer."
  ;; NOTE: For now, just schedule a full reparse.
  ;;       To be implemented later.
  (semantic-parse-tree-set-needs-rebuild))

(defun semantic-matlab-expand-tag (tag)
  "Expand the MATLAB function tag TAG."
  (let ((chil (semantic-tag-components-with-overlays tag)))
    (if chil
        (semantic-tag-put-attribute
         tag :members (mapcar 'semantic-matlab-expand-tag chil)))
    (car (semantic--tag-expand tag))))

(defun semantic-matlab-parse-functions ()
  "Parse all functions from the current MATLAB buffer."
  (car
   (semantic-matlab-sort-raw-tags (semantic-matlab-function-tags)
                                  (point-max))))

(defun semantic-matlab-sort-raw-tags (tag-list &optional end)
  "Return a split list of tags from TAG-LIST before END.
Return list is:
  (TAGS-BEFORE-END REMAINING-TAGS)"
  (let ((newlist nil)
        (rest tag-list))
    ;; Loop until there are no more tags, or no tags before END.
    (while (and tag-list (> end (car (car tag-list))))
      (let* ((tag (car tag-list))
             (start (car tag))
             (end (nth 1 tag))
             (ret (nth 2 tag))
             (name (nth 3 tag))
             (args (nth 4 tag))
             (doc (nth 5 tag))
             (builtin (nth 6 tag))
             (parts (semantic-matlab-sort-raw-tags (cdr tag-list) end))
             (chil (car parts)))
        (setq rest (car (cdr parts)))
        (setq newlist
              (cons (append
                     (semantic-tag-new-function name nil args
                                                :return ret
                                                :subfunctions chil
                                                :documentation doc
                                                :builtin builtin)
                     (list start end))
                    newlist))
        (setq tag-list rest)))
    (list (nreverse newlist) tag-list)))

;; The following function tries to parse MATLAB variable
;; assignments. There are only three categories of types: doubles,
;; structs, and classes. It returns a list with elements
;; (NAME TYPE ATTRIBUTES), for example:
;; ("astruct" "struct" "field1" "field2" "field3")
;; ("aclass" "class" "exampleclass")
;; ("anumber" "double" "1356")
;; Of course we can't parse things we don't know, e.g.
;; if (a==5) variable=aclass; else variable=anotherclass;
;; In these cases, the latter assignment counts.
;; Also, we don't know return types of functions (yet...) - the parser
;; will always think it's "double".  You can override the
;; parser with a special comment, like: %type% avariable = aclass

;; One day we might use a grammar for this...?

(defconst semantic-matlab-type-hint-string "%type%"
  "Comment string which prefixes a type hint for the parser.")

(defun semantic-matlab-parse-assignments ()
  "Parse assignments in current buffer.
This function starts at current point and goes backwards, until
it reaches a function declaration or the beginning of the buffer.
It returns a list of variable assignments (NAME TYPE ATTRIBUTES),
where NAME is unique."
  (let ((limit (or (save-excursion
                     (if (re-search-backward semantic-matlab-match-function-re nil t)
                         (progn
                           (forward-line 1)
                           (point))
                       nil))
                   (point-min)))
        vars)
    ;; don't parse current line
    (beginning-of-line)
    (while (re-search-backward (concat "^\\(" (regexp-quote semantic-matlab-type-hint-string)
                                       "\\)?\\([^%]*[^=><~]\\)=\\([^=].*\\)$") limit t)
      (let ((left (match-string-no-properties 2))
            (right (match-string-no-properties 3))
            temp)
        ;; first we have to deal with elipsis...
        (save-excursion
          (while (string-match
                  (concat "\\(.*\\)"
                          (regexp-quote matlab-elipsis-string) "\\s-*$")
                  right)
            (forward-line 1)
            (setq right
                  (concat
                   (match-string 1 right)
                   (progn (looking-at "^.*$")
                          (match-string-no-properties 0))))))
        (save-excursion
          (while (and (not (bobp))
                      (progn
                        (forward-line -1)
                        (looking-at
                         (concat "\\(.*\\)"
                                 (regexp-quote matlab-elipsis-string) "\\s-*$"))))
            (setq left
                  (concat (match-string-no-properties 1) left))))
        ;; remove bracket expressions and beginning/trailing whitespaces on left-hand side
        (while (or (string-match "\\((.*)\\|{.*}\\)" left)
                   (string-match "^\\(\\s-+\\)" left)
                   (string-match "\\(\\s-+\\)$" left))
          (setq left (replace-match "" t t left)))
        ;; deal with right-hand side
        (cond
          ;; special case: a = set(class,attribute,value)
          ((string-match "\\s-*set(\\s-*\\([A-Za-z_0-9 ]+\\)\\s-*," right)
           (setq right (match-string 1 right)))
          ;; method call which returns same class: class=method(class [,args])
          ((and (string-match "\\s-*[A-Za-z_0-9 ]+\\s-*(\\s-*\\([A-Za-z_0-9 ]+\\)\\s-*\\(,\\|)\\)" right)
                (string= left (match-string 1 right)))
           (setq right (match-string 1 right)))
          ;; otherwise reduce right-hand side to first symbol
          (t
           (string-match "[[({ ]*\\([A-Za-z_0-9]*\\)" right)
           (setq right (match-string 1 right))))
        (cond
          ;; multiple assignment, e.g. [a,b]=size(A);
          ((string-match "\\[\\(.*\\)\\]" left)
           (dolist (cur (split-string (match-string 1 left) ","))
             (string-match "\\s-*\\([A-Za-z_0-9]+\\)\\s-*" cur)
             (setq cur (match-string 1 cur))
             (unless (assoc cur vars)
               ;; since we don't know any return types, we just say it's double
               (push (list cur "double" "") vars))))
          ;; (nested) structure
          ((string-match "\\([A-Za-z_0-9.]+\\)\\.\\([A-Za-z_0-9]+\\)" left)
           (while (string-match "\\([A-Za-z_0-9.]+\\)\\.\\([A-Za-z_0-9]+\\)" left)
             (let ((name (match-string 1 left))
                   (field (match-string 2 left)))
               (if (setq temp (assoc name vars))
                   (unless (member field temp)
                     (nconc temp (list field)))
                 (push (list name "struct" field)
                       vars))
               (setq left name))))
          ;; class
          ((assoc right semanticdb-matlab-user-class-cache)
           (string-match "\\([A-Za-z_0-9]+\\)\\s-*$" left)
           (setq left (match-string 1 left))
           (if (and (setq temp (assoc left vars))
                    (string= (nth 1 temp) "struct"))
               ;; we first thought it's a structure, but it's probably a
               ;; new-style class
               (setcdr temp `("class" ,right))
             (unless temp
               (push `(,left "class" ,right) vars))))
          (t
           ;; default is double
           (string-match "\\([A-Za-z_0-9]+\\)\\s-*$" left)
           (setq left (match-string 1 left))
           (unless (or (assoc left vars)
                       (string= left right)) ; self assignment
             (push `(,left "double" ,right) vars))))))
    vars))



(define-mode-local-override semantic-get-local-variables
    matlab-mode (&optional point)
    "Return a list of local variables for POINT."
    (semanticdb-matlab-cache-files)
    (save-excursion
      (let ((vars (semantic-matlab-parse-assignments))
            knowntypes tags)
        (dolist (cur vars)
          ;; check right-hand side for known types which might be
          ;; assigned this variable
          (if (string= (nth 1 cur) "double")
              (when (member (nth 2 cur) knowntypes)
                (setcdr cur (cdr (assoc (nth 2 cur) vars))))
            (push (nth 0 cur) knowntypes))
          ;; generate the tag
          (push
           (semantic-tag-new-variable
            (car cur)
            (cond
              ;; structures
              ((string= (cadr cur) "struct")
               (semantic-tag-new-type
                ;; this is just a placeholder
                (concat (car cur) "_struct")
                "struct"
                (mapcar
                 (lambda (x)
                   (semantic-tag-new-variable
                    x
                    nil nil :typemodifiers '("public")))
                 (nthcdr 2 cur))
                nil))
              ;; classes
              ((string= (cadr cur) "class")
               ;; include tags from class constructor
               ;; (contains whole class, including methods)
               (car (semanticdb-file-stream
                     (concat
                      (nth 1 (assoc (nth 2 cur) semanticdb-matlab-user-class-cache))
                      "/@" (nth 2 cur) "/" (nth 2 cur) ".m"))))
              (t
               nil)))
           tags))
        tags)))


(define-mode-local-override semantic-tag-components-with-overlays
    matlab-mode (tag)
    "Return the list of subfunctions in TAG."
    (semantic-tag-get-attribute tag :subfunctions))

(define-mode-local-override semantic-format-tag-prototype matlab-mode
  (tag &optional parent color)
  "Return a prototype string describing tag.
For MATLAB, we have to mark builtin functions, since we currently
cannot derive an argument list for them."
  (let ((class (semantic-tag-class tag))
        (name (semantic-format-tag-name tag parent color))
        str)
    (if (eq class 'function)
        (let* ((args (semantic-tag-function-arguments tag))
               (argstr (semantic--format-tag-arguments args
                                                       #'identity
                                                       color))
               (builtin (semantic-tag-get-attribute tag :builtin))
               (doc (semantic-tag-docstring tag)))
          (if builtin
              (if color
                  (setq builtin (semantic--format-colorize-text " [builtin] " 'keyword)
                        argstr (semantic--format-colorize-text " arguments unavailable" 'label))
                (setq builtin " [builtin] "
                      argstr " arguments unavailable"))
            (setq builtin ""))
          (concat name builtin "(" (if args " " "")
                  argstr
                  " )"))
      (semantic-format-tag-prototype-default tag parent color))))

(defun semantic-idle-summary-format-matlab-mode (tag &optional parent color)
  "Describe TAG and display corresponding MATLAB 'lookfor' doc-string."
  (let* ((proto (semantic-format-tag-prototype-matlab-mode tag nil color))
         (doc (semantic-tag-docstring tag)))
    (concat proto " (" doc ")")))

(defcustom-mode-local-semantic-dependency-system-include-path
    matlab-mode semantic-matlab-dependency-system-include-path
  (if semantic-matlab-root-directory
      (mapcar (lambda (cur)
                (concat (file-name-as-directory semantic-matlab-root-directory)
                        cur))
              semantic-matlab-system-paths-include)
    nil)
  "The system include paths from MATLAB.")

(defvar-mode-local matlab-mode semantic-idle-summary-function
  'semantic-idle-summary-format-matlab-mode
  "Function to use when displaying tag information during idle time.")

(defvar semantic-matlab-display-docstring t
  "Flag if function documentation should be displayed after completion.")

(define-mode-local-override semantic-ia-insert-tag
    matlab-mode (tag)
    "Insert TAG into the current buffer based on completion."
    (insert (semantic-tag-name tag))
    (let ((name (semantic-tag-name tag))
          (tt (semantic-tag-class tag))
          (args (semantic-tag-function-arguments tag))
          (doc (semantic-tag-docstring tag)))
      (when (and (eq tt 'function)
                 args
                 (not (looking-at "\\s-*(")))
        (insert "("))
      ;; delete trailing whitespaces when completing class methods
      (when (looking-at "\\(\\s-+\\)(")
        (delete-char (length (match-string 1))))
      (when semantic-matlab-display-docstring
        (fame-message-nolog
         (semantic-idle-summary-format-matlab-mode tag nil t)))))

(define-mode-local-override semantic-ctxt-current-symbol
    matlab-mode (&optional point)
    "Return the current symbol the cursor is on at point in a list.
This will include a list of type/field names when applicable."
    (let* ((case-fold-search semantic-case-fold)
           sym)
      (with-syntax-table semantic-lex-syntax-table
        (save-excursion
          (when point (goto-char point))
          ;; go to beginning of symbol
          (skip-syntax-backward "w_")
          (setq sym (if (looking-at "[a-zA-Z_0-9]+")
                        (match-string-no-properties 0)
                      nil))
          (if sym
              (cond
                ;; method call: var = method(class,args)
                ((progn
                   (and (looking-back "[^=><~]=\\s-*")
                        (looking-at "[a-zA-Z_0-9]*\\s-*(\\([a-zA-Z_0-9]+\\),?")))
                 (list (match-string-no-properties 1) sym))
                ;; class properties: var = get(class,'attribute')
                ((looking-back "\\(get\\|set\\)(\\s-*\\([a-zA-Z_0-9]+\\),'")
                 (list (match-string-no-properties 2) sym))
                ;; (nested) structures or new-style classes
                ((looking-back "[^A-Za-z_0-9.]\\([A-Za-z_0-9.]+\\)\\.")
                 (list (match-string-no-properties 1) sym))
                (t
                 (list sym)))
            nil)))))

 (define-mode-local-override semantic-ctxt-current-symbol-and-bounds
     matlab-mode (&optional point)
     "Return the current symbol and bounds the cursor is on at POINT.
 Uses `semantic-ctxt-current-symbol' to calculate the symbol.
 Return (PREFIX ENDSYM BOUNDS)."
     (let ((sym (semantic-ctxt-current-symbol-matlab-mode point))
           bounds endsym)
       (save-excursion
         (when point (goto-char point))
         (when sym
           ;; go to beginning of symbol
           (skip-syntax-backward "w_")
           (setq endsym (progn (looking-at "[a-zA-Z_0-9]+")
                               (match-string-no-properties 0)))
           (setq bounds (cons
                         (match-beginning 0)
                         (match-end 0)))
           (list sym endsym bounds)))))


;;;###autoload
(defun semantic-default-matlab-setup ()
  "Set up a buffer for parsing of MATLAB files."
  ;; This will use our parser.
  (semantic-install-function-overrides
   '((parse-region . semantic-matlab-parse-region)
     (parse-changes . semantic-matlab-parse-changes)))
  (setq semantic-parser-name "MATLAB"
        ;; Setup a dummy parser table to enable parsing!
        semantic--parse-table t
        imenu-create-index-function 'semantic-create-imenu-index
        ;; semantic-command-separation-character "."
        semantic-type-relation-separator-character '(".")
        semantic-symbol->name-assoc-list '((function . "Function"))
        semantic-imenu-expandable-tag-classes '(function)
        semantic-imenu-bucketize-file nil
        semantic-imenu-bucketize-type-members nil
        senator-step-at-start-end-tag-classes '(function)
        semantic-stickyfunc-sticky-classes '(function)))

(provide 'semantic-matlab)

;;; semantic-matlab.el ends here