Initial commit

services/datasources/spec/unit/arcgis_spec.rb

@@ -0,0 +1,521 @@
+require 'active_support/core_ext'
+
+require_relative '../../lib/datasources'
+require_relative '../doubles/user'
+
+include CartoDB::Datasources
+
+describe Url::ArcGIS do
+
+  before(:all) do
+    @url = 'http://myserver/arcgis/rest/services/MyFakeService/featurename'
+    @invalid_url = 'http://myserver/mysite/rest/myfakefolder/MyFakeService/featurename'
+    @user = CartoDB::Datasources::Doubles::User.new
+  end
+
+  before(:each) do
+    Typhoeus::Expectation.clear
+  end
+
+  describe '#set_data_from' do
+    it 'tests preparing the correct url from the one given from the UI' do
+      invalid_1 = 'http://myserver/services/MyFakeService/featurename/MapServer'
+      invalid_2 = 'myserver/services/MyFakeService/featurename/MapServer'
+
+      test1 = 'http://myserver/arcgis/rest/services/MyFakeService/featurename/MapServer'
+      test2 = 'http://myserver/arcgis/rest/services/MyFakeService/featurename/MapServer/'
+      test3 = 'http://myserver/arcgis/rest/services/MyFakeService/featurename/MapServer/0'
+      test4 = 'http://myserver/arcgis/rest/services/MyFakeService/featurename/MapServer/0?'
+      test5 = 'http://myserver/arcgis/rest/services/MyFakeService/featurename/MapServer/0?q=blablabla'
+      test6 = 'http://myserver/arcgis/rest/services/MyFakeService/featurename/MapServer?q=blablabla'
+
+      valid_map = 'http://myserver/arcgis/rest/services/MyFakeService/featurename/MapServer'
+      valid_map_trailing_slash = 'http://myserver/arcgis/rest/services/MyFakeService/featurename/MapServer/'
+      valid_layer = 'http://myserver/arcgis/rest/services/MyFakeService/featurename/MapServer/0'
+
+      # Should be treated as ok
+      test7   = 'http://myserver/arcgis/rest/services/MyFakeService/featurename/2314/'
+      valid_7 = 'http://myserver/arcgis/rest/services/MyFakeService/featurename/2314/'
+
+      arcgis = Url::ArcGIS.get_new(@user)
+
+      expect {
+        arcgis.send(:sanitize_id, invalid_1)
+      }.to raise_error InvalidInputDataError
+      expect {
+        arcgis.send(:sanitize_id, invalid_2)
+      }.to raise_error InvalidInputDataError
+
+      arcgis.send(:sanitize_id, test1).should eq valid_map
+      arcgis.send(:sanitize_id, test2).should eq valid_map_trailing_slash
+      arcgis.send(:sanitize_id, test3).should eq valid_layer
+      arcgis.send(:sanitize_id, test4).should eq valid_layer
+      arcgis.send(:sanitize_id, test5).should eq valid_layer
+      arcgis.send(:sanitize_id, test6).should eq valid_map
+      arcgis.send(:sanitize_id, test7).should eq valid_7
+    end
+  end
+
+  describe '#get_resource_metadata' do
+    it 'tests error scenarios' do
+      arcgis = Url::ArcGIS.get_new(@user)
+
+      sub_id = '0'
+
+      # 'general http error (non-200)'
+      Typhoeus.stub(/\/arcgis\/rest\/services\/MyFakeService\/featurename\/layers/) do
+        Typhoeus::Response.new(
+          code: 400,
+          headers: { 'Content-Type' => 'application/json' },
+          body: ''
+        )
+      end
+
+      expect {
+        arcgis.get_resource_metadata(@url)
+      }.to raise_error DataDownloadError
+
+      # Stub layers request (so now works)
+      Typhoeus.stub(/\/arcgis\/rest\/services\/MyFakeService\/featurename\/layers/) do |request|
+        body = File.read(File.join(File.dirname(__FILE__), "../fixtures/arcgis_layers.json"))
+        Typhoeus::Response.new(
+          code: 200,
+          headers: { 'Content-Type' => 'application/json' },
+          body: ::JSON.dump(::JSON.parse(body))
+        )
+      end
+
+      layers_data = arcgis.get_resource_metadata(@url)
+      layers_data_expected = {
+        id:           @url,
+        subresources: [{
+          id: "#{@url}/#{sub_id}",
+          title: 'first layer'
+        }]
+      }
+      layers_data.should eq layers_data_expected
+
+      # 'fields' part
+      Typhoeus.stub(/\/arcgis\/rest\/services\/MyFakeService\/featurename\/0/) do |request|
+        body = File.read(File.join(File.dirname(__FILE__), "../fixtures/arcgis_metadata_minimal.json"))
+
+        body = ::JSON.parse(body)
+        body.delete('fields')
+        Typhoeus::Response.new(
+          code: 200,
+          headers: { 'Content-Type' => 'application/json' },
+          body: ::JSON.dump(body)
+        )
+      end
+
+      expect {
+        arcgis.send(:get_subresource_metadata, @url, sub_id)
+      }.to raise_error ResponseError
+
+      # Another required field
+      Typhoeus.stub(/\/arcgis\/rest\/services\/MyFakeService\/featurename\/0/) do |request|
+        body = File.read(File.join(File.dirname(__FILE__), "../fixtures/arcgis_metadata_minimal.json"))
+
+        body = ::JSON.parse(body)
+        body.delete('name')
+        Typhoeus::Response.new(
+          code: 200,
+          headers: { 'Content-Type' => 'application/json' },
+          body: ::JSON.dump(body)
+        )
+      end
+
+      expect {
+        arcgis.send(:get_subresource_metadata, @url, sub_id)
+      }.to raise_error ResponseError
+
+      # Invalid ArcGIS version
+      Typhoeus.stub(/\/arcgis\/rest\/services\/MyFakeService\/featurename\/0/) do |request|
+        body = File.read(File.join(File.dirname(__FILE__), "../fixtures/arcgis_metadata_minimal.json"))
+
+        body = ::JSON.parse(body)
+        body['currentVersion'] = 9.0
+        Typhoeus::Response.new(
+          code: 200,
+          headers: { 'Content-Type' => 'application/json' },
+          body: ::JSON.dump(body)
+        )
+      end
+
+      expect {
+        arcgis.send(:get_subresource_metadata, @url, sub_id)
+      }.to raise_error InvalidServiceError
+
+      # Invalid ArcGIS URL
+      expect {
+        arcgis.send(:get_resource_metadata, @invalid_url)
+      }.to raise_error InvalidInputDataError
+    end
+
+    it 'tests metadata retrieval' do
+      arcgis = Url::ArcGIS.get_new(@user)
+
+      # Stub layers request (so now works)
+      Typhoeus.stub(/\/arcgis\/rest\/services\/MyFakeService\/featurename\/layers/) do |request|
+        body = File.read(File.join(File.dirname(__FILE__), "../fixtures/arcgis_layers.json"))
+        Typhoeus::Response.new(
+          code: 200,
+          headers: { 'Content-Type' => 'application/json' },
+          body: ::JSON.dump(::JSON.parse(body))
+        )
+      end
+
+      Typhoeus.stub(/\/arcgis\/rest\/services\/MyFakeService\/featurename\/0/) do |request|
+        accept = (request.options[:headers]||{})['Accept'] || 'application/json'
+        format = accept.split(',').first
+
+        body = File.read(File.join(File.dirname(__FILE__), "../fixtures/arcgis_metadata_minimal.json"))
+
+        Typhoeus::Response.new(
+          code: 200,
+          headers: { 'Content-Type' => format },
+          body: body
+        )
+      end
+
+      expected_metadata = {
+        :arcgis_version=>10.22,
+        :name=>"Test Feature",
+        :description=>"Sample metadata payload",
+        :type=>"Feature Layer",
+        :geometry_type=>"esriGeometryPolygon",
+        :copyright=>"CartoDB",
+        :fields=>[
+          {
+            :name => "OBJECTID",
+            :type => "esriFieldTypeOID"
+          },
+          {
+            :name => "NAME",
+            :type => "esriFieldTypeString"
+          }
+        ],
+        :max_records_per_query=>1000,
+        :supported_formats=>["JSON", "AMF"],
+        :advanced_queries_supported=>true
+      }
+
+      expected_metadata_response = {
+        id:       @url + '/0',
+        title:    'Test Feature',
+        url:      nil,
+        service:  Url::ArcGIS::DATASOURCE_NAME,
+        checksum: nil,
+        size:     0,
+        filename: 'test_feature.json'
+      }
+
+      # Multi-resource scenario already tested above
+      response = arcgis.get_resource_metadata(@url + '/0')
+
+      response.nil?.should be false
+      arcgis.metadata.should eq expected_metadata
+
+      response.should eq expected_metadata_response
+    end
+  end
+
+  describe '#get_resource' do
+    it 'tests the get_ids_list() private method with error scenarios' do
+      arcgis = Url::ArcGIS.get_new(@user)
+
+      id = arcgis.send(:sanitize_id, @url)
+
+      # 'general http error (non-200)'
+      Typhoeus.stub(/\/arcgis\/rest\//) do
+        Typhoeus::Response.new(
+          code: 400,
+          headers: { 'Content-Type' => 'application/json' },
+          body: ''
+        )
+      end
+
+      expect {
+        arcgis.send(:get_ids_list, id)
+      }.to raise_error DataDownloadError
+
+      # 'objectIds' not present
+      Typhoeus::Expectation.clear
+      Typhoeus.stub(/\/arcgis\/rest\//) do
+        body = File.read(File.join(File.dirname(__FILE__), "../fixtures/arcgis_ids_list.json"))
+
+        body = ::JSON.parse(body)
+        body.delete('objectIds')
+        Typhoeus::Response.new(
+          code: 200,
+          headers: { 'Content-Type' => 'application/json' },
+          body: ::JSON.dump(body)
+        )
+      end
+
+      expect {
+        arcgis.send(:get_ids_list, id)
+      }.to raise_error ResponseError
+
+      # 'objectIds' empty
+      Typhoeus::Expectation.clear
+      Typhoeus.stub(/\/arcgis\/rest\//) do
+        body = File.read(File.join(File.dirname(__FILE__), "../fixtures/arcgis_ids_list.json"))
+
+        body = ::JSON.parse(body)
+        body['objectIds'] = []
+        Typhoeus::Response.new(
+          code: 200,
+          headers: { 'Content-Type' => 'application/json' },
+          body: ::JSON.dump(body)
+        )
+      end
+
+      expect {
+        arcgis.send(:get_ids_list, id)
+      }.to raise_error ResponseError
+
+    end
+
+    it 'tests the get_ids_list() private method' do
+      arcgis = Url::ArcGIS.get_new(@user)
+
+      id = arcgis.send(:sanitize_id, @url)
+
+      Typhoeus.stub(/\/arcgis\/rest\//) do |request|
+        body = File.read(File.join(File.dirname(__FILE__), "../fixtures/arcgis_ids_list.json"))
+        Typhoeus::Response.new(
+          code: 200,
+          headers: { 'Content-Type' => 'application/json' },
+          body: body
+        )
+      end
+
+      expected_ids = [1,2,3,4,5,6,7,8,9,10]
+
+      respose_ids = arcgis.send(:get_ids_list, id)
+
+      respose_ids.nil?.should be false
+      respose_ids.should eq expected_ids
+    end
+
+    it 'tests the get_ids_list() private method on out-of-order ids' do
+      arcgis = Url::ArcGIS.get_new(@user)
+
+      id = arcgis.send(:sanitize_id, @url)
+
+      Typhoeus.stub(/\/arcgis\/rest\//) do |request|
+        body = File.read(File.join(File.dirname(__FILE__), "../fixtures/arcgis_unordered_ids_list.json"))
+        Typhoeus::Response.new(
+          code: 200,
+          headers: { 'Content-Type' => 'application/json' },
+          body: body
+        )
+      end
+
+      expected_ids = [1,2,3,4,5,6,7,8,9,10]
+
+      respose_ids = arcgis.send(:get_ids_list, id)
+
+      respose_ids.nil?.should be false
+      respose_ids.should eq expected_ids
+    end
+
+    it 'tests the get_by_ids() private method with error scenarios' do
+      arcgis = Url::ArcGIS.get_new(@user)
+
+      id = arcgis.send(:sanitize_id, @url)
+
+      # Empty ids
+      expect {
+        arcgis.send(:get_by_ids, id, [], [{ key: 'value' }])
+      }.to raise_error InvalidInputDataError
+
+      # Empty fields
+      expect {
+        arcgis.send(:get_by_ids, id, [1], [])
+      }.to raise_error InvalidInputDataError
+
+
+      # 'general http error (non-200)'
+      Typhoeus.stub(/\/arcgis\/rest\//) do
+        Typhoeus::Response.new(
+          code: 400,
+          headers: { 'Content-Type' => 'application/json' },
+          body: ''
+        )
+      end
+
+      expect {
+        arcgis.send(:get_by_ids, id, [1], [{ key: 'value' }])
+      }.to raise_error DataDownloadError
+    end
+
+    it 'tests the get_by_ids() private method' do
+      arcgis = Url::ArcGIS.get_new(@user)
+
+      id = arcgis.send(:sanitize_id, @url)
+
+      Typhoeus.stub(/\/arcgis\/rest\//) do
+        body = File.read(File.join(File.dirname(__FILE__), "../fixtures/arcgis_data_01.json"))
+        Typhoeus::Response.new(
+          code: 200,
+          headers: { 'Content-Type' => 'application/json' },
+          body: body
+        )
+      end
+
+      expected_response_data = {
+        geometryType: "esriGeometryPolygon",
+        spatialReference: {
+          "wkid" => 4326,
+          "latestWkid" => 4326
+        },
+        fields: [
+          {
+            "name" => "OBJECTID",
+            "type" => "esriFieldTypeOID",
+            "alias" => "OBJECTID"
+          },
+          {
+            "name" => "WDPAID",
+            "type" => "esriFieldTypeInteger",
+            "alias" => "WDPAID"
+          },
+          {
+            "name" => "NAME",
+            "type" => "esriFieldTypeString",
+            "alias" => "NAME",
+            "length" => 254
+          }
+        ],
+        features: [
+          {"attributes"=>{"OBJECTID"=>1, "NAME"=>"Name of object 1"}, "geometry"=>{"fake"=>"geom"}},
+          {"attributes"=>{"OBJECTID"=>2, "NAME"=>"Name of object 2"}, "geometry"=>{"fake"=>"geom"}},
+          {"attributes"=>{"OBJECTID"=>3, "NAME"=>"Name of object 3"}, "geometry"=>{"fake"=>"geom"}}
+        ]
+      }
+
+      ids_to_retrieve = [1,2,3]
+      # WDPAID also present, but left on purpose untouched
+      fields_to_retrieve = [{
+                              name: 'OBJECTID',
+                              type: 'esriFieldTypeOID'
+                            },
+                            {
+                              name: 'NAME',
+                              type: 'esriFieldTypeString'
+                            }]
+
+      response_data = arcgis.send(:get_by_ids, id, ids_to_retrieve, fields_to_retrieve)
+
+      response_data.nil?.should eq false
+      response_data[:features].length.should eq 3
+
+      response_data.should eq expected_response_data
+
+    end
+
+    it 'tests retrieval of data' do
+      arcgis = Url::ArcGIS.get_new(@user)
+
+      feature_names = []
+
+      # Layers request
+      Typhoeus.stub(/\/arcgis\/rest\/services\/MyFakeService\/featurename\/layers/) do |request|
+        body = File.read(File.join(File.dirname(__FILE__), "../fixtures/arcgis_layers.json"))
+        Typhoeus::Response.new(
+          code: 200,
+          headers: { 'Content-Type' => 'application/json' },
+          body: ::JSON.dump(::JSON.parse(body))
+        )
+      end
+
+      # Metadata of a layer
+      Typhoeus.stub(/\/arcgis\/rest\/services\/MyFakeService\/featurename\/0\?f=json/) do
+        body = File.read(File.join(File.dirname(__FILE__), "../fixtures/arcgis_metadata_minimal.json"))
+        Typhoeus::Response.new(
+          code: 200,
+          headers: { 'Content-Type' => 'application/json' },
+          body: body
+        )
+      end
+
+      # IDs list of a layer
+      Typhoeus.stub(/\/arcgis\/rest\/(.*)query\?where=/) do
+        body = File.read(File.join(File.dirname(__FILE__), "../fixtures/arcgis_ids_list_01.json"))
+        Typhoeus::Response.new(
+          code: 200,
+          headers: { 'Content-Type' => 'application/json' },
+          body: body
+        )
+      end
+
+      Typhoeus.stub(/\/arcgis\/rest\/(.*)query$/) do |response|
+        if response.options[:body][:objectIds].to_i == 1
+          # First item fetch of a layer
+          body = File.read(File.join(File.dirname(__FILE__), "../fixtures/arcgis_data_01.json"))
+          body = ::JSON.parse(body)
+
+          feature_names.push body['features'][0]['attributes']['NAME']
+          body['features'] = [ body['features'][0] ]
+        else
+          # Remaining items fetch of a layer, will not use :objectIds
+          body = File.read(File.join(File.dirname(__FILE__), "../fixtures/arcgis_data_01.json"))
+          body = ::JSON.parse(body)
+
+          feature_names.push body['features'][1]['attributes']['NAME']
+          feature_names.push body['features'][2]['attributes']['NAME']
+          body['features'] = [ body['features'][1], body['features'][2] ]
+        end
+
+        Typhoeus::Response.new(
+          code: 200,
+          headers: { 'Content-Type' => 'application/json' },
+          body: ::JSON.dump(body)
+        )
+      end
+
+      # 1) Retrieve lists of layers
+      metadata = arcgis.get_resource_metadata(@url)
+
+      # 2) Retrieve metadata of a specific layer (doesn't adds much, but to replicate flow)
+      item_metadata = arcgis.get_resource_metadata(metadata[:subresources].first[:id])
+
+      item_metadata.nil?.should eq false
+      # No more checks as will fail later if missing something
+
+      spatial_ref_expectation = {"wkid"=>4326, "latestWkid"=>4326}
+
+      initial_stream_data = arcgis.initial_stream(item_metadata[:id])
+
+      initial_stream_data.nil?.should eq false
+      initial_stream_data = ::JSON.parse(initial_stream_data)
+      initial_stream_data['geometryType'].should eq 'esriGeometryPolygon'
+      initial_stream_data['spatialReference'].should eq spatial_ref_expectation
+      initial_stream_data['fields'].count.should eq 3
+      initial_stream_data['fields'][0]['name'].should eq 'OBJECTID'
+      initial_stream_data['fields'][1]['name'].should eq 'WDPAID'
+      initial_stream_data['fields'][2]['name'].should eq 'NAME'
+      initial_stream_data['features'].count.should eq 1
+      initial_stream_data['features'][0]['attributes'].nil?.should eq false
+      initial_stream_data['features'][0]['attributes']['NAME'].should eq feature_names[0]
+      initial_stream_data['features'][0]['geometry'].nil?.should eq false
+
+      streamed_data = arcgis.stream_resource(item_metadata[:id])
+
+      streamed_data.nil?.should eq false
+      streamed_data = ::JSON.parse(streamed_data)
+      streamed_data['geometryType'].should eq 'esriGeometryPolygon'
+      streamed_data['spatialReference'].should eq spatial_ref_expectation
+      streamed_data['fields'].count.should eq 3
+      streamed_data['fields'][0]['name'].should eq 'OBJECTID'
+      streamed_data['fields'][1]['name'].should eq 'WDPAID'
+      streamed_data['fields'][2]['name'].should eq 'NAME'
+      streamed_data['features'].count.should eq 2
+      streamed_data['features'][0]['attributes']['NAME'].should eq feature_names[1]
+      streamed_data['features'][1]['attributes']['NAME'].should eq feature_names[2]
+
+    end
+  end
+end

services/datasources/spec/unit/box_spec.rb

@@ -0,0 +1,31 @@
+require_relative '../../lib/datasources'
+require_relative '../doubles/user'
+
+include CartoDB::Datasources
+
+describe Url::Box do
+  def get_config
+    {
+      'box_host' => '',
+      'application_name' => '',
+      'client_id' => '',
+      'client_secret' => '',
+      'callback_url' => ''
+    }
+  end
+
+  describe '#filters' do
+    it 'test that filter sets correctly' do
+      user_mock = CartoDB::Datasources::Doubles::User.new
+
+      box_provider = Url::Box.get_new(get_config, user_mock)
+
+      box_provider.filter.should eq nil
+
+      # Filter to 'documents'
+      formats = ['csv', 'xls']
+      box_provider.filter = formats
+      box_provider.filter.should eq formats
+    end
+  end
+end

services/datasources/spec/unit/dropbox_spec.rb

@@ -0,0 +1,47 @@
+require_relative '../../lib/datasources'
+require_relative '../doubles/user'
+
+include CartoDB::Datasources
+
+describe Url::Dropbox do
+
+  def get_config
+    {
+      'app_key' => '',
+      'app_secret' => '',
+      'callback_url' => ''
+    }
+  end #get_config
+
+  describe '#filters' do
+    it 'test that filter options work correctly' do
+      user_mock = CartoDB::Datasources::Doubles::User.new
+
+      dropbox_provider = Url::Dropbox.get_new(get_config, user_mock)
+
+      # No filter = all formats allowed
+      filter = []
+      Url::Dropbox::FORMATS_TO_SEARCH_QUERIES.each do |id, search_queries|
+        search_queries.each do |search_query|
+          filter = filter.push(search_query)
+        end
+      end
+      dropbox_provider.filter.should eq filter
+
+      # Filter to 'documents'
+      filter = []
+      format_ids = [ Url::Dropbox::FORMAT_CSV, Url::Dropbox::FORMAT_EXCEL ]
+      Url::Dropbox::FORMATS_TO_SEARCH_QUERIES.each do |id, search_queries|
+        if format_ids.include?(id)
+          search_queries.each do |search_query|
+            filter = filter.push(search_query)
+          end
+        end
+      end
+      dropbox_provider.filter = format_ids
+      dropbox_provider.filter.should eq filter
+    end
+  end #run
+
+end
+

services/datasources/spec/unit/gdrive_spec.rb

@@ -0,0 +1,53 @@
+require_relative '../../../../spec/rspec_configuration'
+require_relative '../../lib/datasources'
+require_relative '../doubles/user'
+
+include CartoDB::Datasources
+
+describe Url::GDrive do
+
+  def get_config
+    {
+      'application_name' => '',
+      'client_id' => '',
+      'client_secret' => '',
+      'callback_url' => 'http://localhost/callback'
+    }
+  end #get_config
+
+  describe '#filters' do
+    it 'test that filter options work correctly' do
+      # Stubs Google Drive client for connectionless testing
+      Google::Apis::DriveV2::DriveService.any_instance.stubs(:get_file)
+      Google::Apis::DriveV2::DriveService.any_instance.stubs(:export_file)
+      Google::Apis::DriveV2::DriveService.any_instance.stubs(:list_files)
+      user_mock = CartoDB::Datasources::Doubles::User.new
+
+      gdrive_provider = Url::GDrive.get_new(get_config, user_mock)
+
+      # No filter = all formats allowed
+      filter = []
+      Url::GDrive::FORMATS_TO_MIME_TYPES.each do |id, mime_types|
+        mime_types.each do |mime_type|
+          filter = filter.push(mime_type)
+        end
+      end
+      gdrive_provider.filter.should eq filter
+
+      # Filter to 'documents'
+      filter = []
+      format_ids = [ Url::GDrive::FORMAT_CSV, Url::GDrive::FORMAT_EXCEL ]
+      Url::GDrive::FORMATS_TO_MIME_TYPES.each do |id, mime_types|
+        if format_ids.include?(id)
+          mime_types.each do |mime_type|
+            filter = filter.push(mime_type)
+          end
+        end
+
+      end
+      gdrive_provider.filter = format_ids
+      gdrive_provider.filter.should eq filter
+    end
+  end #run
+
+end

services/datasources/spec/unit/twitter_spec.rb

@@ -0,0 +1,372 @@
+require_relative '../../lib/datasources'
+require_relative '../doubles/organization'
+require_relative '../doubles/user'
+require_relative '../doubles/search_tweet'
+require_relative '../doubles/data_import'
+require_relative '../../../../lib/cartodb/logger'
+
+include CartoDB::Datasources
+
+describe Search::Twitter do
+
+  def get_config
+    {
+      'auth_required' => false,
+      'username'      => '',
+      'password'      => '',
+      'search_url'    => 'http://fakeurl.carto',
+    }
+  end #get_config
+
+  before(:each) do
+    Typhoeus::Expectation.clear
+  end
+
+  describe '#filters' do
+    it 'tests max and total results filters' do
+      big_quota = 123456
+      user = CartoDB::Datasources::Doubles::User.new({
+        twitter_datasource_quota: big_quota
+      })
+      twitter_datasource = Search::Twitter.get_new(get_config, user)
+
+      maxresults_filter = twitter_datasource.send :build_maxresults_field, user
+      maxresults_filter.should eq CartoDB::TwitterSearch::SearchAPI::MAX_PAGE_RESULTS
+      totalresults_filter = twitter_datasource.send :build_total_results_field, user
+      totalresults_filter.should eq big_quota
+
+      small_quota = 13
+      user = CartoDB::Datasources::Doubles::User.new({
+        twitter_datasource_quota: small_quota,
+        soft_twitter_datasource_limit: false
+      })
+      twitter_datasource = Search::Twitter.get_new(get_config, user)
+      maxresults_filter = twitter_datasource.send :build_maxresults_field, user
+      maxresults_filter.should eq small_quota
+      totalresults_filter = twitter_datasource.send :build_total_results_field, user
+      totalresults_filter.should eq small_quota
+
+      user = CartoDB::Datasources::Doubles::User.new({
+        twitter_datasource_quota: small_quota,
+        soft_twitter_datasource_limit: true
+      })
+      maxresults_filter = twitter_datasource.send :build_maxresults_field, user
+      maxresults_filter.should eq CartoDB::TwitterSearch::SearchAPI::MAX_PAGE_RESULTS
+      totalresults_filter = twitter_datasource.send :build_total_results_field, user
+      totalresults_filter.should eq Search::Twitter::NO_TOTAL_RESULTS
+    end
+
+    it 'tests category filters' do
+      user_mock = CartoDB::Datasources::Doubles::User.new
+      twitter_datasource = Search::Twitter.get_new(get_config, user_mock)
+
+      input_terms = terms_fixture
+
+      expected_output_terms = [
+          {
+            Search::Twitter::CATEGORY_NAME_KEY  => 'Category 1',
+            Search::Twitter::CATEGORY_TERMS_KEY => '(uno OR @dos OR #tres) (has:geo OR has:profile_geo)'
+          },
+          {
+            Search::Twitter::CATEGORY_NAME_KEY  => 'Category 2',
+            Search::Twitter::CATEGORY_TERMS_KEY => '(aaa OR bbb) (has:geo OR has:profile_geo)'
+          }
+      ]
+
+      output = twitter_datasource.send :build_queries_from_fields, input_terms
+
+      output.should eq expected_output_terms
+    end
+
+    it 'tests search term cut if too many' do
+      user_mock = CartoDB::Datasources::Doubles::User.new
+      twitter_datasource = Search::Twitter.get_new(get_config, user_mock)
+
+      input_terms = {
+        categories: [
+          {
+            category: 'Category 1',
+            terms:    Array(1..35)
+          }
+        ]
+      }
+
+      expected_output_terms = [
+          {
+              Search::Twitter::CATEGORY_NAME_KEY  => 'Category 1',
+              Search::Twitter::CATEGORY_TERMS_KEY => '(1 OR 2 OR 3 OR 4 OR 5 OR 6 OR 7 OR 8 OR 9 OR 10 OR 11 OR 12 OR 13 OR 14 OR 15 OR 16 OR 17 OR 18 OR 19 OR 20 OR 21 OR 22 OR 23 OR 24 OR 25 OR 26 OR 27 OR 28 OR 29) (has:geo OR has:profile_geo)'
+          },
+      ]
+
+      output = twitter_datasource.send :build_queries_from_fields, input_terms
+      output.should eq expected_output_terms
+    end
+
+    it 'tests search term cut if too big (even if amount is ok)' do
+      user_mock = CartoDB::Datasources::Doubles::User.new
+      twitter_datasource = Search::Twitter.get_new(get_config, user_mock)
+
+      input_terms = {
+          categories: [
+              {
+                  category: 'Category 1',
+                  terms:    ['wadus1', 'wadus2', 'wadus3' * 500]
+              }
+          ]
+      }
+
+      expect {
+        output = twitter_datasource.send :build_queries_from_fields, input_terms
+
+      }.to raise_error ParameterError
+    end
+
+
+    it 'tests date filters' do
+      user_mock = CartoDB::Datasources::Doubles::User.new
+      twitter_datasource = Search::Twitter.get_new(get_config, user_mock)
+
+      input_dates = dates_fixture
+
+      output = twitter_datasource.send :build_date_from_fields, input_dates, 'from'
+      output.should eq '201403031349'
+
+      output = twitter_datasource.send :build_date_from_fields, input_dates, 'to'
+      output.should eq '201403041159'
+
+      expect {
+        twitter_datasource.send :build_date_from_fields, input_dates, 'wadus'
+      }.to raise_error ParameterError
+
+
+      current_time = Time.now.utc
+      output = twitter_datasource.send :build_date_from_fields, {
+        dates: {
+          toDate:   current_time.strftime("%Y-%m-%d"),
+          toHour:   current_time.hour + 1,  # Set into the future
+          toMin:    current_time.min
+        }
+      }, 'to'
+      output.should eq nil
+
+    end
+
+    it 'tests twitter search integration (without conversion to CSV)' do
+      # This test bridges lots of internal calls to simulate only up until twitter search call and results
+      user_mock = CartoDB::Datasources::Doubles::User.new
+      twitter_datasource = Search::Twitter.get_new(get_config, user_mock)
+
+      input_terms = terms_fixture
+      input_dates = dates_fixture
+
+      Typhoeus.stub(/fakeurl\.carto/) do |request|
+        accept = (request.options[:headers]||{})['Accept'] || 'application/json'
+        format = accept.split(',').first
+
+        if request.options[:params][:next].nil?
+          body = data_from_file('sample_tweets.json')
+        else
+          body = data_from_file('sample_tweets_2.json')
+        end
+
+        Typhoeus::Response.new(
+            code: 200,
+            headers: { 'Content-Type' => format },
+            body: body
+        )
+      end
+
+      twitter_api_config = twitter_datasource.send :search_api_config
+      twitter_api = CartoDB::TwitterSearch::SearchAPI.new(twitter_api_config)
+
+      fields = {
+        categories: input_terms[:categories],
+        dates:      input_dates[:dates]
+      }
+      filters = {
+          Search::Twitter::FILTER_CATEGORIES =>    (twitter_datasource.send :build_queries_from_fields, fields),
+          Search::Twitter::FILTER_FROMDATE =>      (twitter_datasource.send :build_date_from_fields, fields, 'from'),
+          Search::Twitter::FILTER_TODATE =>        (twitter_datasource.send :build_date_from_fields, fields, 'to'),
+          Search::Twitter::FILTER_MAXRESULTS =>    500,
+          Search::Twitter::FILTER_TOTAL_RESULTS => Search::Twitter::NO_TOTAL_RESULTS
+      }
+
+      category = {
+          name:  input_terms[:categories].first[:category],
+          terms: input_terms[:categories].first[:terms],
+      }
+      csv_dumper = twitter_datasource.send :csv_dumper
+      csv_dumper.begin_dump(input_terms[:categories][0][:category])
+      csv_dumper.begin_dump(input_terms[:categories][1][:category])
+      csv_dumper.additional_fields = { category[:name] => category }
+
+      output = twitter_datasource.send :search_by_category, twitter_api, filters, category
+
+      # 2 pages of 10 results per category search
+      output.should eq 20
+
+      csv_dumper.send :destroy_files
+    end
+
+    it 'tests stopping search if runs out of quota' do
+      # Should equal to sample_tweets_3.json number of results, and always >= 10 (because is Gnip's minimum)
+      remaining_tweets_quota = 11
+
+      user_mock = CartoDB::Datasources::Doubles::User.new(
+        twitter_datasource_quota: remaining_tweets_quota
+      )
+      twitter_datasource = Search::Twitter.get_new(get_config, user_mock)
+
+      input_terms = terms_fixture
+      input_dates = dates_fixture
+
+      Typhoeus.stub(/fakeurl\.carto/) do |request|
+        accept = (request.options[:headers]||{})['Accept'] || 'application/json'
+        format = accept.split(',').first
+
+        if request.options[:params][:next].nil?
+          # This dataset has 11 items and a "next"
+          body = data_from_file('sample_tweets_3.json')
+        else
+          body = data_from_file('sample_tweets_2.json')
+        end
+
+        Typhoeus::Response.new(
+            code: 200,
+            headers: { 'Content-Type' => format },
+            body: body
+        )
+      end
+
+      twitter_api_config = twitter_datasource.send :search_api_config
+      twitter_api = CartoDB::TwitterSearch::SearchAPI.new(twitter_api_config)
+
+      fields = {
+          categories: input_terms[:categories],
+          dates:      input_dates[:dates]
+      }
+      filters = {
+          Search::Twitter::FILTER_CATEGORIES =>    (twitter_datasource.send :build_queries_from_fields, fields),
+          Search::Twitter::FILTER_FROMDATE =>      (twitter_datasource.send :build_date_from_fields, fields, 'from'),
+          Search::Twitter::FILTER_TODATE =>        (twitter_datasource.send :build_date_from_fields, fields, 'to'),
+          Search::Twitter::FILTER_MAXRESULTS =>    500,
+          Search::Twitter::FILTER_TOTAL_RESULTS => Search::Twitter::NO_TOTAL_RESULTS
+      }
+
+      category = {
+          name:  input_terms[:categories].first[:category],
+          terms: input_terms[:categories].first[:terms],
+      }
+      csv_dumper = twitter_datasource.send :csv_dumper
+      csv_dumper.begin_dump(input_terms[:categories][0][:category])
+      csv_dumper.begin_dump(input_terms[:categories][1][:category])
+      csv_dumper.additional_fields = { category[:name] => category }
+
+      output = twitter_datasource.send :search_by_category, twitter_api, filters, category
+
+      output.should eq remaining_tweets_quota
+
+      csv_dumper.send :destroy_files
+    end
+
+    it 'tests user limits on datasource usage' do
+      user_mock = CartoDB::Datasources::Doubles::User.new
+      twitter_datasource = Search::Twitter.get_new(get_config, user_mock)
+
+      # Service enabled tests
+      result = twitter_datasource.send :is_service_enabled?, CartoDB::Datasources::Doubles::User.new({
+        has_org: true,
+        twitter_datasource_enabled: true,
+        org_twitter_datasource_enabled: true
+      })
+      result.should eq true
+
+      result = twitter_datasource.send :is_service_enabled?, CartoDB::Datasources::Doubles::User.new({
+        has_org: true,
+        twitter_datasource_enabled: false,
+        org_twitter_datasource_enabled: true
+      })
+      result.should eq false
+
+      result = twitter_datasource.send :is_service_enabled?, CartoDB::Datasources::Doubles::User.new({
+        twitter_datasource_enabled: true,
+      })
+      result.should eq true
+
+      result = twitter_datasource.send :is_service_enabled?, CartoDB::Datasources::Doubles::User.new({
+      twitter_datasource_enabled: false,
+      })
+      result.should eq false
+
+      # Quota & soft limit tests
+      result = twitter_datasource.send :has_enough_quota?, CartoDB::Datasources::Doubles::User.new({
+        soft_twitter_datasource_limit: false,
+        twitter_datasource_quota: 10,
+      })
+      result.should eq true
+
+      result = twitter_datasource.send :has_enough_quota?, CartoDB::Datasources::Doubles::User.new({
+        soft_twitter_datasource_limit: true,
+        twitter_datasource_quota: 0,
+      })
+      result.should eq true
+
+      result = twitter_datasource.send :has_enough_quota?, CartoDB::Datasources::Doubles::User.new({
+        soft_twitter_datasource_limit: false,
+        twitter_datasource_quota: 0,
+      })
+      result.should eq false
+    end
+
+    it 'checks terms sanitize method' do
+      user_mock = CartoDB::Datasources::Doubles::User.new
+      twitter_datasource = Search::Twitter.get_new(get_config, user_mock)
+
+      terms = [ 'a', ' b', 'c ', ' d ', ' e f', 'g h ', ' i j ', ' 1 2 3 4 ', ' ' ]
+      terms_expected = [ 'a', 'b', 'c', 'd', '"e f"', '"g h"', '"i j"', '"1 2 3 4"' ]
+
+      result = twitter_datasource.send :sanitize_terms, terms
+      result.should eq terms_expected
+    end
+
+  end
+
+  protected
+
+  def terms_fixture
+    {
+      categories: [
+        {
+          category: 'Category 1',
+          terms:    ['uno', '@dos', '#tres']
+        },
+        {
+          category: 'Category 2',
+          terms:    ['aaa', 'bbb']
+        }
+      ]
+    }
+  end
+
+  def dates_fixture
+    {
+      dates: {
+        fromDate: '2014-03-03',
+        fromHour: '13',
+        fromMin:  '49',
+        toDate:   '2014-03-04',
+        toHour:   '11',
+        toMin:    '59'
+      }
+    }
+  end
+
+  def data_from_file(filename, fullpath=false)
+    if fullpath
+      File.read(filename)
+    else
+      File.read(File.join(File.dirname(__FILE__), "../fixtures/#{filename}"))
+    end
+  end
+
+end