ENH: Warn instead of raising an error when a request has callbacks.

HISTORY.rst

@@ -10,10 +10,10 @@ History
 0.3.0 (2016-06-24)
 ------------------
 
-* **Backward incompatible change**: Added more restrictions to the request
-  object (no callback/errback).
 * Cleanup callback/errback attributes before sending back the request to the
-  generator.
+  generator. This fixes an edge case when using ``request.repalce()``.
+* Warn if the callback returns requests with callback or errback set.
+* Added deprecation about decorating non-spider functions.
 * Simplified example spider.
 
 0.2.0 (2016-06-23)

src/inline_requests/__init__.py

@@ -42,7 +42,7 @@ def parse(self, response):
     * The decorated method must be a spider method.
     * The decorated method must use the ``yield`` keyword or return a generator.
     * The decorated method must accept ``response`` as the first argument.
-    * The decorated method must yield ``Request`` objects without neither
+    * The decorated method should yield ``Request`` objects without neither
       ``callback`` nor ``errback`` set.
 
     If your requests don't come back to the generator try setting the flag to

src/inline_requests/generator.py

@@ -1,26 +1,48 @@
+import logging
+import warnings
+
 from functools import partial
 from types import GeneratorType
 
 from scrapy.http import Request
 from scrapy.utils.spider import iterate_spider_output
 
 
+logger = logging.getLogger(__name__)
+
+
 class RequestGenerator(object):
     """This is the core class that wraps the callback and outputs the requests
     one by one.
     """
 
     def __init__(self, callback, **kwargs):
+        """Initialize RequestGenerator.
+
+        Parameters
+        ----------
+        callback : callable
+            Callable callback (spider method).
+        **kwargs :
+            Extra callback keyword arguments.
+
+        """
         self.callback = callback
         self.kwargs = kwargs
 
     def __call__(self, response):
+        """Main response entry point.
+
+        This method calls the callback and wraps the returned generator.
+
+        """
         output = iterate_spider_output(self.callback(response=response, **self.kwargs))
         if not isinstance(output, GeneratorType):
             raise ValueError("Callback must return a generator type")
         return self._unwindGenerator(output)
 
     def _unwindGenerator(self, generator, _prev=None):
+        """Unwind (resume) generator."""
         while True:
             if _prev:
                 ret, _prev = _prev, None
@@ -29,11 +51,24 @@ def _unwindGenerator(self, generator, _prev=None):
                     ret = next(generator)
                 except StopIteration:
                     break
+
             if isinstance(ret, Request):
-                yield self._wrapRequest(ret, generator)
-                break
-            else:
-                yield ret
+                if ret.callback:
+                    warnings.warn("Got a request with callback set, bypassing "
+                                  "the generator wrapper. Generator may not "
+                                  "be able to resume. %s" % ret)
+                elif ret.errback:
+                    # By Scrapy defaults, a request without callback defaults to
+                    # self.parse spider method.
+                    warnings.warn("Got a request with errback set, bypassing "
+                                  "the generator wrapper. Generator may not "
+                                  "be able to resume. %s" % ret)
+                else:
+                    yield self._wrapRequest(ret, generator)
+                    return
+
+            # A request with callbacks, item or None object.
+            yield ret
 
     def _wrapRequest(self, request, generator):
         # Allowing existing callback or errbacks could lead to undesired