Initial commit

spec/models/synchronization/collection_spec.rb

@@ -0,0 +1,61 @@
+require_relative '../../spec_helper'
+require_relative '../../../services/data-repository/backend/sequel'
+require_relative '../../../services/data-repository/repository'
+require_relative '../../../app/models/synchronization/collection'
+require_relative '../../../app/models/synchronization/member'
+require 'helpers/unique_names_helper'
+
+include UniqueNamesHelper
+include CartoDB
+
+describe Synchronization::Collection do
+
+  before(:each) do
+  end
+
+  describe '#fetch' do
+    it 'fetches the members of a collection' do
+      user_id = random_uuid
+      Synchronization::Member.new(random_attributes(name: 'sync_1',
+                                                    user_id: user_id)).store
+      Synchronization::Member.new(random_attributes(name: 'sync_2',
+                                                    user_id: user_id)).store
+
+      collection    = Synchronization::Collection.new
+      records       = collection.fetch(user_id: user_id)
+      records.first.name.should == 'sync_1'
+      records.to_a.last.name.should == 'sync_2'
+    end
+  end
+
+  describe '#fetch_many' do
+    it 'fetches many members of the collection to see that paging works' do
+      user_id = random_uuid
+      (1..400).each do |idx|
+        Synchronization::Member.new(random_attributes(name:   "sync_#{idx}",
+                                                      user_id: user_id)).store
+      end
+
+      collection = Synchronization::Collection.new
+      records = collection.fetch(user_id: user_id)
+      records.first.name.should == 'sync_1'
+      records.to_a.last.name.should == 'sync_300'
+      records.count.should == 300
+
+      records = collection.fetch(user_id: user_id, per_page: 900)
+      records.first.name.should == 'sync_1'
+      records.to_a.last.name.should == 'sync_400'
+      records.count.should == 400
+    end
+  end
+
+  def random_attributes(attributes = {})
+    random = unique_integer
+    {
+      name:       attributes.fetch(:name, "name #{random}"),
+      interval:   attributes.fetch(:interval, 900),
+      state:      attributes.fetch(:state, 'enabled'),
+      user_id:    attributes.fetch(:user_id, random_uuid)
+    }
+  end
+end # Synchronization::Collection

spec/models/synchronization/member_overviews_spec.rb

@@ -0,0 +1,176 @@
+require_relative '../../spec_helper'
+
+require_relative '../../../services/data-repository/backend/sequel'
+require_relative '../../../services/data-repository/repository'
+require_relative '../../../app/models/synchronization/member'
+require 'helpers/unique_names_helper'
+require 'helpers/file_server_helper'
+require_relative '../../helpers/feature_flag_helper'
+
+include UniqueNamesHelper
+include CartoDB
+
+describe Synchronization::Member do
+
+  describe "synchronizations" do
+    before(:all) do
+      @user1 = create_user(sync_tables_enabled: true)
+      @user2 = create_user(sync_tables_enabled: true)
+      @feature_flag = FactoryGirl.create(:feature_flag, name: 'create_overviews', restricted: true)
+    end
+
+    before(:each) do
+      bypass_named_maps
+    end
+
+    around(:each) do |example|
+      Cartodb.with_config(metrics: {}, &example)
+    end
+
+    after(:all) do
+      @user1.destroy
+      @user2.destroy
+    end
+
+    include FeatureFlagHelper
+
+    def overview_tables(user, table)
+      overviews = user.in_database do |db|
+        db.fetch %{
+          SELECT * FROM CDB_Overviews('#{table}'::regclass)
+        }
+      end
+      overviews.map(:overview_table)
+    end
+
+    def has_overviews?(user, table)
+      !overview_tables(user, table).empty?
+    end
+
+    def remove_overviews(user, table)
+      user.in_database do |db|
+        db.run %{
+          SELECT CDB_DropOverviews('#{table}'::regclass)
+        }
+      end
+    end
+
+    describe "synchronization" do
+      it 'creates overviews' do
+        CartoDB::Logger.stubs(:error).never
+
+        url = 'https://wadus.com/cities-box.csv'
+        path = "#{Rails.root}/spec/support/data/cities-box.csv"
+        stub_download(url: url, filepath: path, content_disposition: false)
+
+        attrs = random_attributes(user_id: @user1.id).merge(service_item_id: url, url: url, name: 'cities_box')
+        member = Synchronization::Member.new(attrs).store
+
+        set_feature_flag @user1, 'create_overviews', true
+
+        # Import data without overviews needed (too few rows for overviews)
+        data_import = Cartodb.with_config overviews: { 'min_rows' => 1000 } do
+          DataImport.create(
+            user_id: @user1.id,
+            data_source: path,
+            synchronization_id: member.id,
+            service_name: 'public_url',
+            service_item_id: url,
+            updated_at: Time.now,
+            privacy:     ::UserTable::PRIVACY_VALUES_TO_TEXTS.invert['public']
+          ).run_import!
+        end
+        table_name = UserTable[id: data_import.table.id].name
+        has_overviews?(@user1, table_name).should eq false
+
+        # Now synchronize with overviews needed
+        Cartodb.with_config overviews: { 'min_rows' => 100 } do
+          member.run
+        end
+        has_overviews?(@user1, table_name).should eq true
+      end
+
+      it 'deletes overviews' do
+        CartoDB::Logger.stubs(:error).never
+
+        url = 'https://wadus.com/cities-box.csv'
+        path = "#{Rails.root}/spec/support/data/cities-box.csv"
+        stub_download(url: url, filepath: path, content_disposition: false)
+
+        attrs = random_attributes(user_id: @user1.id).merge(service_item_id: url, url: url, name: 'cities_box')
+        member = Synchronization::Member.new(attrs).store
+
+        set_feature_flag @user1, 'create_overviews', true
+
+        # Import data with overviews needed (too few rows for overviews)
+        data_import = Cartodb.with_config overviews: { 'min_rows' => 100 } do
+          DataImport.create(
+            user_id: @user1.id,
+            data_source: path,
+            synchronization_id: member.id,
+            service_name: 'public_url',
+            service_item_id: url,
+            updated_at: Time.now,
+            privacy:     ::UserTable::PRIVACY_VALUES_TO_TEXTS.invert['public']
+          ).run_import!
+        end
+        table_name = UserTable[id: data_import.table.id].name
+        member.name = table_name
+        has_overviews?(@user1, table_name).should eq true
+
+        # Now synchronize without overviews needed
+        Cartodb.with_config overviews: { 'min_rows' => 1000 } do
+          member.run
+        end
+        has_overviews?(@user1, table_name).should eq false
+      end
+
+      it 'updates overviews' do
+        CartoDB::Logger.stubs(:error).never
+
+        url = 'https://wadus.com/cities-box.csv'
+        path = "#{Rails.root}/spec/support/data/cities-box.csv"
+        stub_download(url: url, filepath: path, content_disposition: false)
+
+        attrs = random_attributes(user_id: @user1.id).merge(service_item_id: url, url: url, name: 'cities_box')
+        member = Synchronization::Member.new(attrs).store
+
+        set_feature_flag @user1, 'create_overviews', true
+
+        # Import data with overviews needed (too few rows for overviews)
+        data_import = Cartodb.with_config overviews: { 'min_rows' => 100 } do
+          DataImport.create(
+            user_id: @user1.id,
+            data_source: path,
+            synchronization_id: member.id,
+            service_name: 'public_url',
+            service_item_id: url,
+            updated_at: Time.now,
+            privacy:     ::UserTable::PRIVACY_VALUES_TO_TEXTS.invert['public']
+          ).run_import!
+        end
+        table_name = UserTable[id: data_import.table.id].name
+        has_overviews?(@user1, table_name).should eq true
+
+        # Now synchronize with overviews needed
+        # TODO: alter or delete overviews contents and check they're are re-generated
+        Cartodb.with_config overviews: { 'min_rows' => 100 } do
+          member.run
+        end
+        has_overviews?(@user1, table_name).should eq true
+      end
+    end
+  end
+
+  private
+
+  def random_attributes(attributes = {})
+    random = unique_integer
+    {
+      name:       attributes.fetch(:name, "name#{random}"),
+      interval:   attributes.fetch(:interval, 15 * 60 + random),
+      state:      attributes.fetch(:state, 'enabled'),
+      user_id:    attributes.fetch(:user_id, nil)
+    }
+  end
+end

spec/models/synchronization/member_spec.rb

@@ -0,0 +1,277 @@
+require_relative '../../spec_helper'
+
+require_relative '../../../services/data-repository/backend/sequel'
+require_relative '../../../services/data-repository/repository'
+require_relative '../../../app/models/synchronization/member'
+require 'helpers/unique_names_helper'
+require 'helpers/file_server_helper'
+
+include UniqueNamesHelper
+include CartoDB
+
+describe Synchronization::Member do
+  describe 'Basic actions' do
+    it 'assigns an id by default' do
+      member = Synchronization::Member.new
+      member.should be_an_instance_of Synchronization::Member
+      member.id.should_not be_nil
+    end
+
+    it 'persists attributes to the repository' do
+      attributes  = random_attributes
+      member      = Synchronization::Member.new(attributes)
+      member.store
+
+      member      = Synchronization::Member.new(id: member.id)
+      member.name.should be_nil
+
+      member.fetch
+      member.name             .should == attributes.fetch(:name)
+    end
+
+    it 'fetches attributes from the repository' do
+      attributes  = random_attributes
+      member      = Synchronization::Member.new(attributes).store
+      member      = Synchronization::Member.new(id: member.id)
+      member.name = 'changed'
+      member.fetch
+      member.name.should == attributes.fetch(:name)
+    end
+
+    it 'deletes this member from the repository' do
+      member      = Synchronization::Member.new(random_attributes).store
+      member.fetch
+      member.name.should_not be_nil
+
+      member.delete
+
+      member.name.should be_nil
+      lambda { member.fetch }.should raise_error KeyError
+    end
+  end
+
+  describe "synchronizations" do
+    before(:all) do
+      @user1 = create_user(sync_tables_enabled: true)
+      @user2 = create_user(sync_tables_enabled: true)
+    end
+
+    before(:each) do
+      bypass_named_maps
+    end
+
+    around(:each) do |example|
+      Cartodb.with_config(metrics: {}, &example)
+    end
+
+    after(:all) do
+      @user1.destroy
+      @user2.destroy
+    end
+
+    describe 'external sources' do
+      it "Authorizes to sync always if from an external source" do
+        member = Synchronization::Member.new(random_attributes(user_id: @user1.id)).store
+        member.fetch
+
+        member.expects(:from_external_source?).returns(true)
+
+        @user1.sync_tables_enabled = true
+        @user2.sync_tables_enabled = true
+
+        member.authorize?(@user1).should eq true
+        member.authorize?(@user2).should eq false
+
+        @user1.sync_tables_enabled = false
+        @user2.sync_tables_enabled = false
+
+        member.authorize?(@user1).should eq true
+      end
+    end
+
+    describe "synchronization" do
+      it 'syncs' do
+        # TODO: this is the minimum test valid to reproduce #11889, it's not a complete sync test
+        CartoDB::Logger.stubs(:error).never
+
+        url = 'https://wadus.com/guess_country.csv'
+
+        path = fake_data_path('guess_country.csv')
+        stub_download(url: url, filepath: path, content_disposition: false)
+
+        attrs = random_attributes(user_id: @user1.id).merge(service_item_id: url, url: url, name: 'guess_country')
+        member = Synchronization::Member.new(attrs).store
+
+        DataImport.create(
+          user_id: @user1.id,
+          data_source: fake_data_path('guess_country.csv'),
+          synchronization_id: member.id,
+          service_name: 'public_url',
+          service_item_id: url,
+          updated_at: Time.now
+        ).run_import!
+
+        member.run
+      end
+
+      it 'fails if user is inactive' do
+        url = 'https://wadus.com/guess_country.csv'
+
+        path = fake_data_path('guess_country.csv')
+        stub_download(url: url, filepath: path, content_disposition: false)
+
+        attrs = random_attributes(user_id: @user1.id).merge(service_item_id: url, url: url, name: 'guess_country')
+        member = Synchronization::Member.new(attrs).store
+
+        DataImport.create(
+          user_id: @user1.id,
+          data_source: fake_data_path('guess_country.csv'),
+          synchronization_id: member.id,
+          service_name: 'public_url',
+          service_item_id: url,
+          updated_at: Time.now
+        ).run_import!
+        @user1.state = Carto::User::STATE_LOCKED
+        @user1.save
+
+        CartoDB::Logger.stubs(:error).once
+
+        member.fetch.run
+
+        member.log.entries.should match /Can't run a synchronization for inactive user/
+        expect(member.state).to eq 'failure'
+
+        @user1.state = Carto::User::STATE_ACTIVE
+        @user1.sync_tables_enabled = true
+        @user1.save
+        @user1.reload
+      end
+
+      it 'fails to overwrite tables with views by replacement' do
+        url = 'https://wadus.com/guess_country.csv'
+
+        path = fake_data_path('guess_country.csv')
+        stub_download(url: url, filepath: path, content_disposition: false)
+
+        attrs = random_attributes(user_id: @user2.id).merge(service_item_id: url, url: url, name: 'guess_country')
+        member = Synchronization::Member.new(attrs).store
+
+        DataImport.create(
+          user_id: @user2.id,
+          data_source: path,
+          synchronization_id: member.id,
+          service_name: 'public_url',
+          service_item_id: url,
+          updated_at: Time.now
+        ).run_import!
+
+        @user2.in_database.execute('CREATE VIEW wadus AS SELECT * FROM guess_country')
+
+        member.run
+        expect(member.state).to eq 'failure'
+        expect(member.error_code).to eq 2013
+      end
+
+      it 'it can overwrite tables with views by sync' do
+        url = 'https://wadus.com/guess_country_geocoded.csv'
+
+        path = fake_data_path('guess_country_geocoded.csv')
+        stub_download(url: url, filepath: path, content_disposition: false)
+
+        attrs = random_attributes(user_id: @user2.id).merge(service_item_id: url, url: url, name: 'guess_country_geocoded')
+        member = Synchronization::Member.new(attrs).store
+
+        DataImport.create(
+          user_id: @user2.id,
+          data_source: path,
+          synchronization_id: member.id,
+          service_name: 'public_url',
+          service_item_id: url,
+          updated_at: Time.now
+        ).run_import!
+
+        @user2.in_database.execute('CREATE VIEW wadus_geocoded AS SELECT * FROM guess_country_geocoded')
+
+        member.run
+        expect(member.state).to eq 'success'
+      end
+
+      it 'should sync files with missing ogc_fid' do
+        stub_arcgis_response_with_file(
+          File.expand_path('spec/fixtures/arcgis_response_missing_ogc_fid.json'),
+          File.expand_path('spec/fixtures/arcgis_metadata_ogc_fid.json')
+        )
+
+        url = 'https://wtf.com/arcgis/rest/services/Planning/EPI_Primary_Planning_Layers/MapServer/2'
+
+        attrs = random_attributes(user_id: @user1.id)
+                .merge(service_item_id: url, url: url, name: 'land_zoning')
+        member = Synchronization::Member.new(attrs).store
+
+        data_import = DataImport.create(
+          user_id: @user1.id,
+          synchronization_id: member.id,
+          service_name: 'arcgis',
+          service_item_id: url,
+          updated_at: Time.now
+        )
+
+        data_import.run_import!
+        expect(data_import.state).to eq 'complete'
+
+        source_file = CartoDB::Importer2::SourceFile.new(
+          File.expand_path('spec/fixtures/arcgis_response_missing_ogc_fid.json'),
+          'arcgis_response_missing_ogc_fid.json'
+        )
+        CartoDB::Importer2::Downloader.any_instance.stubs(:download_and_store).returns(source_file)
+        CartoDB::Importer2::Downloader.any_instance.stubs(:source_file).returns(source_file)
+        member.run
+        expect(member.state).to eq 'success'
+      end
+
+      it 'keeps indices' do
+        url = 'https://wadus.com/clubbing.csv'
+
+        path = fake_data_path('clubbing.csv')
+        stub_download(url: url, filepath: path, content_disposition: false)
+
+        attrs = random_attributes(user_id: @user2.id).merge(service_item_id: url, url: url, name: 'clubbing')
+        member = Synchronization::Member.new(attrs).store
+
+        # Create table with index
+        DataImport.create(
+          user_id: @user2.id,
+          data_source: path,
+          synchronization_id: member.id,
+          service_name: 'public_url',
+          service_item_id: url,
+          updated_at: Time.now
+        ).run_import!
+        @user2.in_database.execute('CREATE INDEX ON clubbing (nombre)')
+
+        # Sync the table
+        member.run
+        expect(member.state).to eq 'success'
+
+        # Expect custom and default indices to still exist
+        table = UserTable.where(user: @user2, name: 'clubbing').first
+        indexed_columns = table.service.pg_indexes.map { |x| x[:column] }
+        expected_indices = ['cartodb_id', 'the_geom', 'the_geom_webmercator', 'nombre']
+
+        expect(indexed_columns.sort).to eq(expected_indices.sort)
+      end
+    end
+  end
+
+  private
+
+  def random_attributes(attributes={})
+    random = unique_integer
+    {
+      name:       attributes.fetch(:name, "name#{random}"),
+      interval:   attributes.fetch(:interval, 15 * 60 + random),
+      state:      attributes.fetch(:state, 'enabled'),
+      user_id:    attributes.fetch(:user_id, nil)
+    }
+  end
+end

spec/models/synchronization/synchronization_oauth_spec.rb

@@ -0,0 +1,160 @@
+require 'spec_helper'
+
+describe SynchronizationOauth do
+
+  before(:all) do
+    @user = create_user(:quota_in_bytes => 500.megabytes, :table_quota => 500)
+    @user.reload
+  end
+
+  after(:all) do
+    bypass_named_maps
+    @user.destroy
+  end
+
+  after(:each) do
+    @user.synchronization_oauths.map &:destroy
+    @user.reload
+  end
+
+  context '#creation_updating' do
+    it 'Tests basic creation operations' do
+      service_name = 'testtest'
+      token_value = 'qv2345q235erfaweerfdsdfsds'
+      token_value_2 = 'aaaaaaaaaaaaaaaaaaaaaaaa'
+
+      SynchronizationOauth.all.size.should eq 0
+
+      oauth_entry = SynchronizationOauth.create(
+          user_id: @user.id,
+          service: service_name,
+          token: token_value
+      ).reload
+
+      oauth_entry.nil?.should eq false
+      oauth_entry.user_id.should eq @user.id
+      oauth_entry.service.should eq service_name
+      oauth_entry.token.should eq token_value
+
+      SynchronizationOauth.all.size.should eq 1
+
+      entry_second_handle = SynchronizationOauth.where(user_id: @user.id, service: service_name).first
+      entry_second_handle.nil?.should eq false
+      entry_second_handle.user_id.should eq @user.id
+      entry_second_handle.service.should eq service_name
+      entry_second_handle.token.should eq token_value
+
+      entry_second_handle.should eq oauth_entry
+
+      oauth_entry.token = token_value_2
+      oauth_entry.save.reload
+      oauth_entry.token.should eq token_value_2
+
+      expect {
+        oauth_entry.service = 'another_service_name'
+        oauth_entry.save
+      }.to raise_exception Sequel::ValidationFailed
+      oauth_entry.service = service_name
+
+      expect {
+        oauth_entry.user_id = UUIDTools::UUID.timestamp_create
+        oauth_entry.save
+      }.to raise_exception Sequel::ValidationFailed
+
+      expect {
+        SynchronizationOauth.create(
+            user_id: @user.id,
+            service: service_name,
+            token: UUIDTools::UUID.timestamp_create
+        )
+      }.to raise_exception Sequel::ValidationFailed
+
+      expect {
+        SynchronizationOauth.create(
+          user_id: @user_id,
+          service: service_name,
+          token: nil
+        )
+      }.to raise_exception Sequel::ValidationFailed
+
+      expect {
+        SynchronizationOauth.create(
+          user_id: @user_id,
+          service: service_name,
+          token: ''
+        )
+      }.to raise_exception Sequel::ValidationFailed
+    end
+  end
+
+  context '#deletion' do
+    it 'tests deletion of items' do
+      another_uuid = UUIDTools::UUID.timestamp_create.to_s
+      service_name = 'testtest'
+      service_name_2 = '123456'
+      token_value = 'qv2345q235erfaweerfdsdfsds'
+      token_value_2 = 'aaaaaaaaaaaaaaaaaaaaaaaa'
+
+      SynchronizationOauth.create(
+          user_id: @user.id,
+          service: service_name,
+          token: token_value
+      )
+      SynchronizationOauth.create(
+          user_id: @user.id,
+          service: service_name_2,
+          token: token_value_2
+      )
+      SynchronizationOauth.create(
+          user_id: another_uuid,
+          service: service_name_2,
+          token: token_value_2
+      )
+
+      @user.synchronization_oauths.size.should eq 2
+
+      oauth = SynchronizationOauth.where(user_id: @user.id, service: service_name).first
+      oauth.destroy
+      @user.reload
+      @user.synchronization_oauths.size.should eq 1
+
+      oauth = SynchronizationOauth.where(user_id: @user.id, service: service_name_2).first
+      oauth.destroy
+      @user.reload
+      @user.synchronization_oauths.size.should eq 0
+    end
+  end
+
+  context '#user_oauths' do
+    it 'tests the one to many association of user and sync oauths' do
+      service_name = 'testtest'
+      token_value = 'qv2345q235erfaweerfdsdfsds'
+
+      @user.synchronization_oauths.size.should eq 0
+
+      SynchronizationOauth.create(
+          user_id: @user.id,
+          service: service_name,
+          token: token_value
+      )
+      @user.reload
+      @user.synchronization_oauths.size.should eq 1
+
+      retrieved_entry = @user.synchronization_oauths.first
+      @user.synchronization_oauths.first.should eq retrieved_entry
+
+      @user.synchronization_oauths.first.destroy
+      @user.reload
+      @user.synchronization_oauths.size.should eq 0
+
+      @user.add_synchronization_oauth(service: service_name, token: token_value)
+      @user.synchronization_oauths.size.should eq 1
+
+      expect {
+        @user.add_synchronization_oauth(service: service_name, token: token_value)
+      }.to raise_exception Sequel::ValidationFailed
+
+    end
+  end
+
+end