C parser fails to parse valid file

[Quintus]

Hi there,

RDoc’s C parser fails to parse the following C file (it is part of a library I’m currently working on):

#include "database.h"

/***************************************
 * Variables
 ***************************************/

VALUE rb_cAlpm_Database;

/***************************************
 * Methods
 ***************************************/

static VALUE initialize(VALUE self)
{
  rb_raise(rb_eNotImpError, "Can't create new databases with this library.");
  return self;
}

/**
 * call-seq:
 *   name() → a_string
 *
 * Returns the name of the package database.
 */
static VALUE name(VALUE self)
{
  alpm_db_t* p_db = NULL;
  Data_Get_Struct(self, alpm_db_t, p_db);

  return rb_str_new2(alpm_db_get_name(p_db));
}

/**
 * call-seq:
 *   get( name ) → a_package
 *
 * Find a Package by name in the database.
 *
 * === Parameters
 * [name]
 *   The name of the package. Must match exactly.
 *
 * === Return value
 * A Package instance representing the package.
 */
static VALUE get(VALUE self, VALUE name)
{
  alpm_db_t* p_db = NULL;
  alpm_pkg_t* p_pkg = NULL;
  Data_Get_Struct(self, alpm_db_t, p_db);

  p_pkg = alpm_db_get_pkg(p_db, StringValuePtr(name));

  if (p_pkg)
    return Data_Wrap_Struct(rb_cAlpm_Package, NULL, NULL, p_pkg);
  else
    return Qnil;
}

/***************************************
 * Binding
 ***************************************/


/**
 * A Database is the list of packages in a repository, where the notion
 * of the "repository" is an abstract one. It may be an actual remote
 * repository, or just represent the current system state. It does not
 * make any difference in treating instances of this class.
 *
 * The database listings are always kept local, so interacting with this
 * class is possible while being offline.
 */
void Init_database()
{
  rb_cAlpm_Database = rb_define_class_under(rb_cAlpm, "Database", rb_cObject);

  rb_define_singleton_method(rb_cAlpm_Database, "get", RUBY_METHOD_FUNC(get), 1);

  rb_define_method(rb_cAlpm_Database, "initialize", RUBY_METHOD_FUNC(initialize), 0);
  rb_define_method(rb_cAlpm_Database, "name", RUBY_METHOD_FUNC(name), 0);
}

RDoc does not crash on parsing, but it simply reports a class named unknown with no methods or description at all instead of the expected Alpm::Database class. rb_cAlpm is defined in another file main.c if this is important:

/* ... */
void Init_alpm()
{
  rb_cAlpm = rb_define_class("Alpm", rb_cObject);
  rb_define_alloc_func(rb_cAlpm, allocate);

  rb_define_method(rb_cAlpm, "initialize", RUBY_METHOD_FUNC(initialize), 2);

  /* ... */

  Init_database();
  /* ... */
}

However, the behaviour is observable even without that accompanying file, i.e. if given as the only file to RDoc for documentation, it still results in an unknown class as shown by the following run with --debug enabled:

Parsing sources...
100% [ 1/ 1]  foo.c                                                             

Generating Darkfish format into /tmp/doc...
Copying static files
cp /opt/rubies/ruby-2.0.0-p0/lib/ruby/gems/2.0.0/gems/rdoc-4.0.1/lib/rdoc/generator/template/darkfish/rdoc.css .
mkdir -p js
cp /opt/rubies/ruby-2.0.0-p0/lib/ruby/gems/2.0.0/gems/rdoc-4.0.1/lib/rdoc/generator/template/darkfish/js/search.js js/search.js
cp /opt/rubies/ruby-2.0.0-p0/lib/ruby/gems/2.0.0/gems/rdoc-4.0.1/lib/rdoc/generator/template/darkfish/js/jquery.js js/jquery.js
cp /opt/rubies/ruby-2.0.0-p0/lib/ruby/gems/2.0.0/gems/rdoc-4.0.1/lib/rdoc/generator/template/darkfish/js/darkfish.js js/darkfish.js
mkdir -p images
cp /opt/rubies/ruby-2.0.0-p0/lib/ruby/gems/2.0.0/gems/rdoc-4.0.1/lib/rdoc/generator/template/darkfish/images/macFFBgHack.png images/macFFBgHack.png
cp /opt/rubies/ruby-2.0.0-p0/lib/ruby/gems/2.0.0/gems/rdoc-4.0.1/lib/rdoc/generator/template/darkfish/images/wrench.png images/wrench.png
cp /opt/rubies/ruby-2.0.0-p0/lib/ruby/gems/2.0.0/gems/rdoc-4.0.1/lib/rdoc/generator/template/darkfish/images/ruby.png images/ruby.png
cp /opt/rubies/ruby-2.0.0-p0/lib/ruby/gems/2.0.0/gems/rdoc-4.0.1/lib/rdoc/generator/template/darkfish/images/bug.png images/bug.png
cp /opt/rubies/ruby-2.0.0-p0/lib/ruby/gems/2.0.0/gems/rdoc-4.0.1/lib/rdoc/generator/template/darkfish/images/bullet_toggle_minus.png images/bullet_toggle_minus.png
cp /opt/rubies/ruby-2.0.0-p0/lib/ruby/gems/2.0.0/gems/rdoc-4.0.1/lib/rdoc/generator/template/darkfish/images/tag_green.png images/tag_green.png
cp /opt/rubies/ruby-2.0.0-p0/lib/ruby/gems/2.0.0/gems/rdoc-4.0.1/lib/rdoc/generator/template/darkfish/images/bullet_black.png images/bullet_black.png
cp /opt/rubies/ruby-2.0.0-p0/lib/ruby/gems/2.0.0/gems/rdoc-4.0.1/lib/rdoc/generator/template/darkfish/images/wrench_orange.png images/wrench_orange.png
cp /opt/rubies/ruby-2.0.0-p0/lib/ruby/gems/2.0.0/gems/rdoc-4.0.1/lib/rdoc/generator/template/darkfish/images/plugin.png images/plugin.png
cp /opt/rubies/ruby-2.0.0-p0/lib/ruby/gems/2.0.0/gems/rdoc-4.0.1/lib/rdoc/generator/template/darkfish/images/brick_link.png images/brick_link.png
cp /opt/rubies/ruby-2.0.0-p0/lib/ruby/gems/2.0.0/gems/rdoc-4.0.1/lib/rdoc/generator/template/darkfish/images/page_green.png images/page_green.png
cp /opt/rubies/ruby-2.0.0-p0/lib/ruby/gems/2.0.0/gems/rdoc-4.0.1/lib/rdoc/generator/template/darkfish/images/page_white_text.png images/page_white_text.png
cp /opt/rubies/ruby-2.0.0-p0/lib/ruby/gems/2.0.0/gems/rdoc-4.0.1/lib/rdoc/generator/template/darkfish/images/page_white_width.png images/page_white_width.png
cp /opt/rubies/ruby-2.0.0-p0/lib/ruby/gems/2.0.0/gems/rdoc-4.0.1/lib/rdoc/generator/template/darkfish/images/bullet_toggle_plus.png images/bullet_toggle_plus.png
cp /opt/rubies/ruby-2.0.0-p0/lib/ruby/gems/2.0.0/gems/rdoc-4.0.1/lib/rdoc/generator/template/darkfish/images/date.png images/date.png
cp /opt/rubies/ruby-2.0.0-p0/lib/ruby/gems/2.0.0/gems/rdoc-4.0.1/lib/rdoc/generator/template/darkfish/images/delete.png images/delete.png
cp /opt/rubies/ruby-2.0.0-p0/lib/ruby/gems/2.0.0/gems/rdoc-4.0.1/lib/rdoc/generator/template/darkfish/images/loadingAnimation.gif images/loadingAnimation.gif
cp /opt/rubies/ruby-2.0.0-p0/lib/ruby/gems/2.0.0/gems/rdoc-4.0.1/lib/rdoc/generator/template/darkfish/images/find.png images/find.png
cp /opt/rubies/ruby-2.0.0-p0/lib/ruby/gems/2.0.0/gems/rdoc-4.0.1/lib/rdoc/generator/template/darkfish/images/brick.png images/brick.png
cp /opt/rubies/ruby-2.0.0-p0/lib/ruby/gems/2.0.0/gems/rdoc-4.0.1/lib/rdoc/generator/template/darkfish/images/zoom.png images/zoom.png
cp /opt/rubies/ruby-2.0.0-p0/lib/ruby/gems/2.0.0/gems/rdoc-4.0.1/lib/rdoc/generator/template/darkfish/images/tag_blue.png images/tag_blue.png
cp /opt/rubies/ruby-2.0.0-p0/lib/ruby/gems/2.0.0/gems/rdoc-4.0.1/lib/rdoc/generator/template/darkfish/images/arrow_up.png images/arrow_up.png
cp /opt/rubies/ruby-2.0.0-p0/lib/ruby/gems/2.0.0/gems/rdoc-4.0.1/lib/rdoc/generator/template/darkfish/images/package.png images/package.png
cp /opt/rubies/ruby-2.0.0-p0/lib/ruby/gems/2.0.0/gems/rdoc-4.0.1/lib/rdoc/generator/template/darkfish/images/add.png images/add.png
cp /opt/rubies/ruby-2.0.0-p0/lib/ruby/gems/2.0.0/gems/rdoc-4.0.1/lib/rdoc/generator/template/darkfish/images/transparent.png images/transparent.png
Rendering the index page...
Outputting to /tmp/doc/index.html
Generating class documentation in /tmp/doc
  working on unknown (unknown.html)
  rendering /tmp/doc/unknown.html
Outputting to /tmp/doc/unknown.html
Generating file documentation in /tmp/doc
  working on foo.c (/tmp/doc/foo_c.html)
Rendering the Table of Contents...
Outputting to /tmp/doc/table_of_contents.html
Generating JSON index
  writing search index to js/search_index.js
  generating class search index
    unknown
  generating method search index
  generating pages search index
mkdir -p /tmp/doc/js
install -c -m 0644 js/navigation.js /tmp/doc/js/navigation.js
install -c -m 0644 js/searcher.js /tmp/doc/js/searcher.js

Files:      1

Classes:    1 (1 undocumented)
Modules:    0 (0 undocumented)
Constants:  0 (0 undocumented)
Attributes: 0 (0 undocumented)
Methods:    0 (0 undocumented)

Total:      1 (1 undocumented)
  0.00% documented

Elapsed: 0.0s

RDoc version: 4.0.1
Ruby version: ruby 2.0.0p0 (2013-02-24 revision 39474) [x86_64-linux]
OS: Arch Linux

Valete,
Marvin

[drbrain]

RDoc does not contain a full C parser.

Since rb_cAlpm is not declared in (presumably) database.c there's no way for RDoc to know what class to attach Database to. One workaround for this is to give RDoc a hint by declaring rb_cAlpm like:

    void Init_database()
    {
      rb_cAlpm = rb_const_get(rb_cObject, "Alpm");
      rb_cAlpm_Database = rb_define_class_under(rb_cAlpm, "Database", rb_cObject);
      …

A second workaround is to wire all your classes and methods to ruby names in a single file, but this requires non-static functions.

If I save the file with Alpm::Database as "database.c" and save the file with Alpm as "alpm.c" RDoc discovers what rb_cAlpm means and links it properly, but it depends on file order which may not be favorable to declaration order (rdoc sorts the file list before parsing). For example, Alpm may be declared in "driver.c" which gives the result you show.

This presents a third workaround, adding a namespace to your files so you have alpm.c declaring Alpm and alpm_database.c declaring Alpm::Database.

A fix for this situation is possible by attempting to resolve C-defined classes with unknown parents after all files are parsed.

[drbrain]

Fixing this bug involves a lot of work in RDoc but is easily worked-around (see previous comment) so I am delaying it to a future version of RDoc.

I created a test which helps illustrate the magnitude of the change needed. The first problem is storing the information needed to create methods, aliases, constants, etc. that will be attached to the not-yet-defined class (or module). On top of this, the unknown classes with the saved content will need to be resolved outside of the single-file parser (as in ld).

This is possible by storing the information needed to create the class and its internals in RDoc::Store and having future runs of the C parser check this information to see if it has learned something new. The current structure of the C parser makes this work a little easier, but not easier enough.

The test:

diff --git a/test/test_rdoc_parser_c.rb b/test/test_rdoc_parser_c.rb
index 29ac0e7..5c238fd 100644
--- a/test/test_rdoc_parser_c.rb
+++ b/test/test_rdoc_parser_c.rb
@@ -1699,6 +1699,31 @@ void Init(void) {
     assert_equal expected, @store.c_singleton_class_variables
   end

+  def test_scan_missing_parent
+    parser = util_parser <<-C
+/*
+ * Returns the name of the package database.
+ */
+static VALUE
+alpm_name(VALUE self) { }
+
+void Init_database()
+{
+    /* rb_cAlpm is included elsewhere */
+    VALUE rb_cAlpm_Database =
+        rb_define_class_under(rb_cAlpm, "Database", rb_cObject);
+
+    rb_define_method(rb_cAlpm_Database, "name", alpm_name, 0);
+}
+    C
+
+    parser.scan
+
+    database = parser.classes.values.first
+
+    assert_includes database.method_list.map { |m| m.name }, 'name'
+  end
+
   def test_scan_method_copy
     parser = util_parser <<-C
 /*