Merge pull request #380 from tdt/development

Development

app/Tdt/Core/DataControllers/CSVController.php

@@ -6,6 +6,9 @@
 use Tdt\Core\Pager;
 use Tdt\Core\Repositories\Interfaces\TabularColumnsRepositoryInterface;
 use Tdt\Core\Repositories\Interfaces\GeoPropertyRepositoryInterface;
+use Tdt\Core\Datasets\DatasetController;
+
+ini_set('auto_detect_line_endings', true);
 
 /**
  * CSV Controller
@@ -34,6 +37,15 @@ public function readData($source_definition, $rest_parameters = array())
     {
         list($limit, $offset) = Pager::calculateLimitAndOffset();
 
+        // Get the format, if it's CSV we allow a full read of the datasource without paging limitation
+        list($uri, $extension) = $this->processURI(\Request::path());
+
+        $ignore_paging = false;
+
+        if (strtolower($extension) == 'csv') {
+            $ignore_paging = true;
+        }
+
         // Disregard the paging when rest parameters are given
         if (!empty($rest_parameters)) {
             $limit = PHP_INT_MAX;
@@ -130,7 +142,7 @@ public function readData($source_definition, $rest_parameters = array())
                             );
 
         if (($handle = fopen($uri, "r", false, stream_context_create($ssl_options))) !== false) {
-            while (($data = fgetcsv($handle, 2000000, $delimiter)) !== false) {
+            while (($data = fgetcsv($handle, 0, $delimiter)) !== false) {
                 if ($total_rows >= $start_row) {
                     // Create the values array, containing the (aliased) name of the column
                     // to the value of a the row which $data represents
@@ -158,12 +170,11 @@ public function readData($source_definition, $rest_parameters = array())
 
                 $total_rows++;
 
-                if ($total_rows >= 10000) {
+                if ($total_rows >= 10000 && !$ignore_paging) {
                     break;
                 }
             }
             fclose($handle);
-
         } else {
             \App::abort(500, "Cannot retrieve any data from the CSV file on location $uri.");
         }
@@ -199,6 +210,41 @@ private function createValues($columns, $data)
         return $result;
     }
 
+    /**
+     * Parse the URI and look for the resource name and the format
+     *
+     * @param string $uri
+     *
+     * @return array
+     */
+    private function processURI($uri)
+    {
+        $dot_position = strrpos($uri, '.');
+
+        if (!$dot_position) {
+            return array($uri, null);
+        }
+
+        // If a dot has been found, do a couple
+        // of checks to find out if it introduces a formatter
+        $uri_parts = explode('.', $uri);
+
+        $possible_extension = strtoupper(array_pop($uri_parts));
+
+        $uri = implode('.', $uri_parts);
+
+        $formatter_class = 'Tdt\\Core\\Formatters\\' . $possible_extension . 'Formatter';
+
+        if (!class_exists($formatter_class)) {
+            // Re-attach the dot with the latter part of the uri
+            $uri .= '.' . strtolower($possible_extension);
+
+            return array($uri, null);
+        }
+
+        return array($uri, strtolower($possible_extension));
+    }
+
     /**
      * Parse the columns from a CSV file and return them
      * Optionally aliases can be given to columns as well as a primary key

app/Tdt/Core/Datasets/DatasetController.php

@@ -280,22 +280,24 @@ private function processURI($uri)
     private static function applyRestFilter($data, $rest_params)
     {
         foreach ($rest_params as $rest_param) {
-            if (is_object($data) && $key = self::propertyExists($data, $rest_param)) {
-                $data = $data->$key;
-            } elseif (is_array($data)) {
-                if ($key = self::keyExists($data, $rest_param)) {
-                    $data = $data[$key];
-                } elseif (is_numeric($rest_param)) {
-                    for ($i = 0; $i <= $rest_param; $i++) {
-                        $result = array_shift($data);
-                    }
+            if (!empty($rest_param)) {
+                if (is_object($data) && $key = self::propertyExists($data, $rest_param)) {
+                    $data = $data->$key;
+                } elseif (is_array($data)) {
+                    if ($key = self::keyExists($data, $rest_param)) {
+                        $data = $data[$key];
+                    } elseif (is_numeric($rest_param)) {
+                        for ($i = 0; $i <= $rest_param; $i++) {
+                            $result = array_shift($data);
+                        }
 
-                    $data = $result;
+                        $data = $result;
+                    } else {
+                        \App::abort(404, "No property ($rest_param) has been found.");
+                    }
                 } else {
                     \App::abort(404, "No property ($rest_param) has been found.");
                 }
-            } else {
-                \App::abort(404, "No property ($rest_param) has been found.");
             }
         }
 

app/Tdt/Core/Repositories/DcatRepository.php

@@ -37,8 +37,6 @@ public function getDcatDocument(array $definitions, $oldest_definition)
         // Create a new EasyRDF graph
         $graph = new \EasyRdf_Graph();
 
-        \EasyRdf_Namespace::set('adms', 'http://www.w3.org/ns/adms#');
-
         $all_settings = $this->settings->getAll();
 
         $uri = \Request::root();
@@ -105,7 +103,7 @@ public function getDcatDocument(array $definitions, $oldest_definition)
 
                 // Backwards compatibility
                 if (!empty($definition['contact_point'])) {
-                    $graph->addResource($dataset_uri, 'adms:contactPoint', $definition['contact_point']);
+                    $graph->addResource($dataset_uri, 'dcat:contactPoint', $definition['contact_point']);
                 }
 
                 // Add the publisher resource to the dataset

app/Tdt/Core/Validators/CustomValidator.php

@@ -10,17 +10,18 @@
  */
 class CustomValidator extends \Illuminate\Validation\Validator
 {
-
     /**
      * Check if the URI can be resolved externally or locally
      */
     public function validateUri($attribute, $value, $parameters)
     {
         try {
-            if (!filter_var($value, FILTER_VALIDATE_URL) === false) {
-                $data = $this->getRemoteData($value);
+            $url_pieces = parse_url($value);
 
-                return !empty($data);
+            if (!filter_var($value, FILTER_VALIDATE_URL) === false && ($url_pieces['scheme'] == 'http' || $url_pieces['scheme'] == 'https')) {
+                $status = $this->getHeadInfo($value);
+
+                return $status == 200;
             } else {
                 $data =@ file_get_contents($value);
 
@@ -31,6 +32,31 @@ public function validateUri($attribute, $value, $parameters)
         }
     }
 
+    private function getHeadInfo($uri)
+    {
+        $c = curl_init();
+        curl_setopt($c, CURLOPT_URL, $uri);
+        curl_setopt($c, CURLOPT_RETURNTRANSFER, 1);
+
+        curl_setopt($c, CURLOPT_SSL_VERIFYHOST, false);
+        curl_setopt($c, CURLOPT_SSL_VERIFYPEER, false);
+
+        curl_setopt($c, CURLOPT_CONNECTTIMEOUT, 9);
+        curl_setopt($c, CURLOPT_TIMEOUT, 60);
+        curl_setopt($c, CURLOPT_CUSTOMREQUEST, 'HEAD');
+        curl_setopt($c, CURLOPT_NOBODY, true);
+
+        curl_exec($c);
+        $status = curl_getinfo($c);
+        curl_close($c);
+
+        if (!empty($status['http_code'])) {
+            return $status['http_code'];
+        } else {
+            return 500;
+        }
+    }
+
     private function getRemoteData($url)
     {
         $c = curl_init();

composer.json

@@ -16,7 +16,7 @@
         }
     },
     "require": {
-        "laravel/framework": "4.2.0",
+        "laravel/framework": "4.2.1",
         "phpoffice/phpexcel": "1.8.0",
         "phpunit/phpunit" : "4.0.17",
         "easyrdf/easyrdf" : "0.9.1",