Initial commit

lib/importer/README.md

@@ -0,0 +1,44 @@
+# CartoDB importer #
+
+CartoDB importer is a Ruby gem that makes your life easier when importing data from a file into a PostGIS database. The accepted formats for input files are:
+
+  - CSV
+  - SHP (in a zip file)
+  - ODS
+  - XLX(S)
+  
+## How to use it? ##
+
+The way to use this gem is to initialize a object of class Cartodb::Importer using the appropiate parameters.
+
+    importer = CartoDB::Importer.new :import_from_file => "path to CSV file", :srid => 4326, :database => "...", :username => "...", :password => "..."
+    result = importer.import!
+  
+If everything works fine, a new table will exist in the given database. A `result` object is return with some information about the import, such as the number of rows, or the name of the table.
+
+    puts result.rows_imported
+    # > 43243
+  
+If any error happens, an exception could be raised.
+
+This is the list with all the available options to use in the constructor:
+
+  - import_from_file: a file descriptor, Tempfile or URL with the URL from which import the data
+  - srid: the value of the SRID 
+  - database: the name of the database where import the data
+  - username: the owner of the database
+  - password: the password to connect to the database
+  - extra_columns: a SQL string with some extra columns that should be added to the imported table. If any of these columns already exists an error will be raised
+  
+## Running the specs ##
+
+CartoDB Importer has a suite of specs which define its specification. To run this suite a database named cartodb_importer_test must exist. You can create this database by running:
+
+    CREATE DATABASE cartodb_importer_test
+    WITH TEMPLATE = template_postgis
+    OWNER = postgres
+
+Then, to run the specs just run this command:
+
+    bundle exec rspec spec/lib/import_spec.rb
+    

lib/importer/lib/cartodb-migrator.rb

@@ -0,0 +1,7 @@
+require 'rubygems'
+require 'csv'
+require 'tempfile'
+require 'ostruct'
+
+# main file last
+require File.dirname(__FILE__) + '/cartodb-migrator/migrator'

lib/importer/lib/cartodb-migrator/migrator.rb

@@ -0,0 +1,118 @@
+module CartoDB
+  class Migrator
+    class << self
+      attr_accessor :debug
+    end
+    @@debug = true
+
+    attr_accessor :current_name, :suggested_name, :db_configuration, :db_connection
+
+    attr_reader :table_created, :force_name
+
+    def initialize(options = {})
+      @@debug = options[:debug] if options[:debug]
+      @table_created = nil
+
+      # Handle name of table and target name of table
+      @suggested_name = options[:current_name]
+      @current_name = options[:current_name]
+
+      @target_schema = options[:schema]
+
+      raise "current_table value can't be nil" if @current_name.nil?
+
+      # Handle DB connection
+      @db_configuration = options.slice :database, :username, :password, :host, :port
+      @db_configuration = {:port => 5432, :host => '127.0.0.1'}.merge @db_configuration
+      @db_connection = Sequel.connect("postgres://#{@db_configuration[:username]}:#{@db_configuration[:password]}@#{@db_configuration[:host]}:#{@db_configuration[:port]}/#{@db_configuration[:database]}")
+
+      #handle suggested_name
+      unless options[:suggested_name].nil? || options[:suggested_name].blank?
+        @force_name = true
+        @suggested_name = options[:suggested_name]
+      else
+        @force_name = false
+      end
+
+    rescue => e
+      log $!
+      log e.backtrace
+      raise e
+    end
+
+    def migrate!
+      # Already done by Table#sanitize_columns in app/models/table.rb
+      #sanitize_columns!
+
+      # Rename our table
+      if @current_name != @suggested_name
+        @db_connection.run("ALTER TABLE #{@current_name} RENAME TO #{@suggested_name}")
+        @current_name = @suggested_name
+      end
+
+      @table_created = true
+
+      return @suggested_name
+
+    rescue => e
+      log "====================="
+      log $!
+      log e.backtrace
+      log "====================="
+      unless @table_created.nil?
+        @db_connection.drop_table(@suggested_name)
+      end
+      raise e
+    ensure
+      @db_connection.disconnect
+    end
+
+    private
+
+    def log(str)
+      if @@debug
+        puts str
+      end
+    end
+
+    # Sanitize column names where needed
+    def sanitize_columns!
+      column_names = @db_connection.schema(@current_name, {:schema => @target_schema}).map{ |s| s[0].to_s }
+      sanitize(column_names)
+    end
+
+
+    def sanitize(column_names)
+      columns_to_sanitize = column_names.select do |column_name|
+        column_name != column_name.sanitize_column_name
+      end
+
+      correct_columns = column_names - columns_to_sanitize
+
+      sanitization_map = Hash[
+        columns_to_sanitize.map { |column_name|
+          [column_name, column_name.sanitize_column_name]
+        }
+      ]
+
+      sanitization_count = 0
+
+      sanitization_map = sanitization_map.inject({}) { |memo, pair|
+        if memo.values.include?(pair.last) || correct_columns.include?(pair.last)
+          sanitization_count += 1
+          memo.merge(pair.first => "#{pair.last}_#{sanitization_count}")
+        else
+          memo.merge(pair.first => pair.last)
+        end
+      }
+
+      sanitization_map.each do |unsanitized, sanitized|
+        @db_connection.run(%Q{
+          ALTER TABLE #{@current_name}
+          RENAME COLUMN "#{unsanitized}"
+          TO "#{sanitized}"
+        })
+      end
+    end
+  end
+end

lib/importer/misc/csv_normalizer.py

@@ -0,0 +1,27 @@
+import sys
+import brewery.ds as ds
+import brewery.dq as dq
+from chardet.universaldetector import UniversalDetector
+
+filename = sys.argv[1]
+
+detector = UniversalDetector()
+for line in file(filename, 'rb'):
+    detector.feed(line)
+    if detector.done: break
+detector.close()
+
+src = ds.CSVDataSource(filename, read_header = True, encoding=detector.result["encoding"], delimiter=',' )
+src.initialize()
+if len(src.field_names) == 1:
+  src.finalize()
+  src = ds.CSVDataSource(filename, read_header = True, encoding=detector.result["encoding"], delimiter=';' )
+  src.initialize()
+  
+out = ds.CSVDataTarget(sys.stdout, encoding='utf-8')
+out.fields = ds.fieldlist(src.field_names)
+out.initialize()
+for record in src.records():
+  out.append(record)
+src.finalize()
+out.finalize()

lib/importer/misc/csv_remove_newlines.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+
+# Remove newline chars from CSV "cells"
+# Input is taken from stdin and output spit to stdout
+
+import csvkit
+import sys
+
+reader = csvkit.reader(sys.stdin)
+writer = csvkit.writer(sys.stdout)
+for row in reader:
+  for i in range(0, len(row)):
+    if isinstance(row[i], (str, unicode)):
+      if "\n" in row[i]:
+        row[i] = row[i].replace("\n", '')
+  writer.writerow(row)

lib/importer/misc/dbfUtils.py

@@ -0,0 +1,121 @@
+import struct, datetime, decimal, itertools
+
+def dbfreader(f):
+    """Returns an iterator over records in a Xbase DBF file.
+
+    The first row returned contains the field names.
+    The second row contains field specs: (type, size, decimal places).
+    Subsequent rows contain the data records.
+    If a record is marked as deleted, it is skipped.
+
+    File should be opened for binary reads.
+
+    """
+    # See DBF format spec at:
+    #     http://www.pgts.com.au/download/public/xbase.htm#DBF_STRUCT
+
+    numrec, lenheader = struct.unpack('<xxxxLH22x', f.read(32))    
+    numfields = (lenheader - 33) // 32
+
+    fields = []
+    for fieldno in xrange(numfields):
+        name, typ, size, deci = struct.unpack('<11sc4xBB14x', f.read(32))
+        name = name.replace('\0', '')       # eliminate NULs from string   
+        fields.append((name, typ, size, deci))
+    yield [field[0] for field in fields]
+    yield [tuple(field[1:]) for field in fields]
+
+    terminator = f.read(1)
+    assert terminator == '\r'
+
+    fields.insert(0, ('DeletionFlag', 'C', 1, 0))
+    fmt = ''.join(['%ds' % fieldinfo[2] for fieldinfo in fields])
+    fmtsiz = struct.calcsize(fmt)
+    for i in xrange(numrec):
+        record = struct.unpack(fmt, f.read(fmtsiz))
+        if record[0] != ' ':
+            continue                        # deleted record
+        result = []
+        for (name, typ, size, deci), value in itertools.izip(fields, record):
+            if name == 'DeletionFlag':
+                continue
+            if typ == "N":
+                value = value.replace('\0', '').lstrip()
+                if value == '':
+                    value = 0
+                elif deci:
+                    try:
+                       value = decimal.Decimal(value)
+                    except decimal.InvalidOperation:
+                       # Could be '************************' (for NULL)
+                       value = 0
+                else:
+                    value = value
+            elif typ == 'D':
+                y, m, d = int(value[:4]), int(value[4:6]), int(value[6:8])
+                try:
+                    value = datetime.date(y, m, d)
+                except ValueError:
+                       # Could be '00000000' (for NULL)
+                       value = 0
+            elif typ == 'L':
+                value = (value in 'YyTt' and 'T') or (value in 'NnFf' and 'F') or '?'
+            result.append(value)
+        yield result
+
+
+def dbfwriter(f, fieldnames, fieldspecs, records):
+    """ Return a string suitable for writing directly to a binary dbf file.
+
+    File f should be open for writing in a binary mode.
+
+    Fieldnames should be no longer than ten characters and not include \x00.
+    Fieldspecs are in the form (type, size, deci) where
+        type is one of:
+            C for ascii character data
+            M for ascii character memo data (real memo fields not supported)
+            D for datetime objects
+            N for ints or decimal objects
+            L for logical values 'T', 'F', or '?'
+        size is the field width
+        deci is the number of decimal places in the provided decimal object
+    Records can be an iterable over the records (sequences of field values).
+    
+    """
+    # header info
+    ver = 3
+    now = datetime.datetime.now()
+    yr, mon, day = now.year-1900, now.month, now.day
+    numrec = len(records)
+    numfields = len(fieldspecs)
+    lenheader = numfields * 32 + 33
+    lenrecord = sum(field[1] for field in fieldspecs) + 1
+    hdr = struct.pack('<BBBBLHH20x', ver, yr, mon, day, numrec, lenheader, lenrecord)
+    f.write(hdr)
+                      
+    # field specs
+    for name, (typ, size, deci) in itertools.izip(fieldnames, fieldspecs):
+        name = name.ljust(11, '\x00')
+        fld = struct.pack('<11sc4xBB14x', name, typ, size, deci)
+        f.write(fld)
+
+    # terminator
+    f.write('\r')
+
+    # records
+    for record in records:
+        f.write(' ')                        # deletion flag
+        for (typ, size, deci), value in itertools.izip(fieldspecs, record):
+            if typ == "N":
+                value = str(value).rjust(size, ' ')
+            elif typ == 'D':
+                value = value.strftime('%Y%m%d')
+            elif typ == 'L':
+                value = str(value)[0].upper()
+            else:
+                value = str(value)[:size].ljust(size, ' ')
+            assert len(value) == size
+            f.write(value)
+
+    # End of file
+    f.write('\x1A')

lib/importer/misc/shp_normalizer.py

@@ -0,0 +1,111 @@
+from chardet.universaldetector import UniversalDetector
+import itertools
+import os.path
+import sys
+import dbfUtils
+import sys
+from osgeo import osr
+from urllib import urlencode
+from urllib2 import urlopen
+import json
+import subprocess
+
+if len(sys.argv) != 3:
+    print "usage: python %s shp_file name" % sys.argv[0]
+    sys.exit()
+shp_file = sys.argv[1]
+name = sys.argv[2]
+
+dbf_file = shp_file[0:-4] + '.dbf'
+prj_file = shp_file[0:-4] + '.prj'
+
+shape_name = os.path.splitext(shp_file)[0]
+
+def get_spatial_reference(shapefile):
+    srs = osr.SpatialReference()
+    shpfile = os.path.basename(shape_name)
+    ret = srs.SetFromUserInput(prj_string)
+    proj4 = srs.ExportToProj4()
+    if not proj4:
+        prj_file = open(shape_name +'.prj','r')
+        prj_lines = prj_file.readlines()
+        prj_file.close()
+        for i in range(len(prj_lines)):
+            prj_lines[i] = prj_string.rstrip( prj_lines[i] )
+        srs = osr.SpatialReference()
+        srs.ImportFromESRI(prj_lines)
+        proj4 = srs.ExportToProj4()
+        """
+        if not proj4:
+            #print 'Failed to convert prj of %s, giving up...' % shpfile
+        else:
+            #print 'Second try assuming ESRI wkt worked for %s!' % shpfile
+        """
+        srs.from_esri = True
+    else:
+        srs.from_esri = False
+    srs.AutoIdentifyEPSG()
+    return srs
+
+def to_epsg(srs):
+    if srs.IsGeographic():
+        return srs.GetAuthorityCode('GEOGCS')
+    else:
+        c = srs.GetAuthorityCode('PROJCS')
+        if c:
+            return c
+        else:
+            try:
+                return srs.GetAuthorityCode('GEOGCS')
+            except:
+                return None
+
+
+srid = None
+
+#Try detecting the SRID
+if os.path.isfile(prj_file):
+  prj_string = open(prj_file,'r').read()
+  code = to_epsg(get_spatial_reference(shp_file))
+
+  srid = code if code else None
+
+try:
+# Try to detect the encoding
+    dbf = open(dbf_file.strip(), 'rb')
+    db = dbfUtils.dbfreader(dbf)
+
+    fnames = db.next()
+    ftypes = db.next()
+
+    # find string fields
+    sfields = []
+    for fno in range(len(fnames)):
+      if ( ftypes[fno][0] == 'C' ) : sfields.append(fno)
+
+    detector = UniversalDetector()
+
+    # TODO: Make this a % of total table size and stop guessing correct values for guessing
+    for row in itertools.islice(db, 1000):
+      # Feed detector with concatenated string fields
+      detector.feed( ''.join(row[fno] for fno in sfields) )
+      if detector.done: break
+    dbf.close()
+    detector.close()
+    encoding = detector.result["encoding"]
+    confidence = detector.result["confidence"]
+    if encoding=="ascii":
+        encoding="LATIN1" # why not UTF8 here ?
+    # There's problems detecting LATIN1 encodings, it detects KOI8-R instead of LATIN1
+    if encoding=="KOI8-R":
+        encoding="LATIN1"
+    # Fix for #1336: since ISO-8859-2 is unlikely and UniversalDetector doesn't support ISO-8859-1, 
+    # we'll fallback to ISO-8859-1 if confidence is not high
+    if encoding=="ISO-8859-2" and confidence < 0.75:
+        encoding="ISO-8859-1"
+except Exception as err:
+    encoding="None" # why not UTF8 here ?
+    #sys.stderr.write(repr(err)+'\n')
+    #sys.exit(1)
+
+print "%s,%s,%s,%s" % (srid,encoding,shp_file,name)

lib/importer/misc/srid_from_gdal.py

@@ -0,0 +1,11 @@
+import sys
+from osgeo import gdal, osr
+
+f = sys.argv[1]
+g = gdal.Open(f)
+proj = g.GetProjection()
+srs = osr.SpatialReference()
+srs.ImportFromESRI([proj])
+srs.AutoIdentifyEPSG()
+srid = srs.GetAuthorityCode(None)
+print "%s" %(srid)