Initial commit

services/datasources/spec/integration/csv_file_dumper_spec.rb

@@ -0,0 +1,51 @@
+require_relative '../../lib/datasources'
+require_relative '../doubles/json_to_csv_converter'
+
+include CartoDB::Datasources
+
+describe CSVFileDumper do
+
+  before(:each) do
+  end
+
+  describe '#dumping data' do
+    it 'tests streaming dump' do
+      output_stream_name ='/tmp/csv_file_dumper_test.csv'
+
+      File.unlink(output_stream_name) if File.exists?(output_stream_name)
+
+      converter_mock = CartoDB::TwitterSearch::Doubles::JSONToCSVConverter.new
+
+      dumper = CSVFileDumper.new(converter_mock, false)
+
+      dumper.buffer_size=2
+
+      input_filename_1 = 'stream_input_1'
+      input_filename_2 = 'stream_input_2'
+      input_data1 = File.read(File.join(File.dirname(__FILE__), "../fixtures/#{input_filename_1}"))
+      input_data2 = File.read(File.join(File.dirname(__FILE__), "../fixtures/#{input_filename_2}"))
+
+      names_list = [ input_filename_1, input_filename_2 ]
+
+      stream = File.open(output_stream_name, 'wb')
+
+      dumper.begin_dump(input_filename_1)
+      dumper.dump(input_filename_1, [input_data1])
+      dumper.end_dump(input_filename_1)
+      dumper.begin_dump(input_filename_2)
+      dumper.dump(input_filename_2, [input_data2])
+      dumper.end_dump(input_filename_2)
+
+      dumper.merge_dumps_into_stream(names_list, stream)
+
+      stream.close
+
+      data = File.read(output_stream_name)
+      data.should eq "\n#{input_data1}\n#{input_data2}\n"
+      File.unlink(output_stream_name)
+
+    end
+  end
+
+end
+

services/datasources/spec/integration/twitter_spec.rb

@@ -0,0 +1,103 @@
+require_relative '../../lib/datasources'
+require_relative '../doubles/organization'
+require_relative '../doubles/user'
+require_relative '../doubles/search_tweet'
+require_relative '../doubles/data_import'
+require_relative '../../../../lib/cartodb/logger'
+
+include CartoDB::Datasources
+
+describe Search::Twitter do
+
+  def get_config
+    {
+      'auth_required' => false,
+      'username'      => '',
+      'password'      => '',
+      'search_url'    => 'http://fakeurl.carto',
+    }
+  end
+
+  before(:each) do
+    Typhoeus::Expectation.clear
+  end
+
+  describe '#search' do
+    it 'tests basic full search flow with streaming' do
+      user_quota = 100
+      user_mock = CartoDB::Datasources::Doubles::User.new({twitter_datasource_quota: user_quota})
+      data_import_mock = CartoDB::Datasources::Doubles::DataImport.new(id: '123456789', service_item_id: '987654321')
+      twitter_datasource = Search::Twitter.get_new(get_config, user_mock)
+
+      input_terms = terms_fixture
+      input_dates = dates_fixture
+
+      Typhoeus.stub(/fakeurl\.carto/) do |request|
+        accept = (request.options[:headers]||{})['Accept'] || 'application/json'
+        format = accept.split(',').first
+
+        request.base_url.should eq 'http://fakeurl.carto'
+        request.options[:params].key?(:pusblisher).should eq false
+        body = data_from_file('sample_tweets_v2.json')
+
+        Typhoeus::Response.new(
+            code: 200,
+            headers: { 'Content-Type' => format },
+            body: body
+        )
+      end
+
+      twitter_datasource.send :audit_entry, CartoDB::Datasources::Doubles::SearchTweet
+      twitter_datasource.data_import_item = data_import_mock
+      stream_location = '/tmp/sample_tweets_v2.csv'
+      File.unlink(stream_location) if File.exists?(stream_location)
+      stream = File.open(stream_location, 'wb')
+      twitter_datasource.stream_resource(::JSON.dump(
+         {
+           categories: input_terms[:categories],
+           dates:      input_dates[:dates]
+         }
+       ), stream)
+      stream.close
+
+      stored_data = data_from_file(stream_location, true)
+      stored_data.should eq data_from_file('sample_tweets_v2.csv')
+      File.unlink(stream_location)
+    end
+  end
+
+  protected
+
+  def terms_fixture
+    {
+      categories: [
+        {
+          category: '1',
+          terms:    ['carto']
+        }
+      ]
+    }
+  end
+
+  def dates_fixture
+    {
+      dates: {
+        fromDate: '2017-02-21',
+        fromHour: '11',
+        fromMin:  '45',
+        toDate:   '2017-02-21',
+        toHour:   '12',
+        toMin:    '00'
+      }
+    }
+  end
+
+  def data_from_file(filename, fullpath=false)
+    if fullpath
+      File.read(filename)
+    else
+      File.read(File.join(File.dirname(__FILE__), "../fixtures/#{filename}"))
+    end
+  end
+
+end