Initial commit

lib/importer/misc/csv_normalizer.py

@@ -0,0 +1,27 @@
+import sys
+import brewery.ds as ds
+import brewery.dq as dq
+from chardet.universaldetector import UniversalDetector
+
+filename = sys.argv[1]
+
+detector = UniversalDetector()
+for line in file(filename, 'rb'):
+    detector.feed(line)
+    if detector.done: break
+detector.close()
+
+src = ds.CSVDataSource(filename, read_header = True, encoding=detector.result["encoding"], delimiter=',' )
+src.initialize()
+if len(src.field_names) == 1:
+  src.finalize()
+  src = ds.CSVDataSource(filename, read_header = True, encoding=detector.result["encoding"], delimiter=';' )
+  src.initialize()
+  
+out = ds.CSVDataTarget(sys.stdout, encoding='utf-8')
+out.fields = ds.fieldlist(src.field_names)
+out.initialize()
+for record in src.records():
+  out.append(record)
+src.finalize()
+out.finalize()

lib/importer/misc/csv_remove_newlines.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+
+# Remove newline chars from CSV "cells"
+# Input is taken from stdin and output spit to stdout
+
+import csvkit
+import sys
+
+reader = csvkit.reader(sys.stdin)
+writer = csvkit.writer(sys.stdout)
+for row in reader:
+  for i in range(0, len(row)):
+    if isinstance(row[i], (str, unicode)):
+      if "\n" in row[i]:
+        row[i] = row[i].replace("\n", '')
+  writer.writerow(row)

lib/importer/misc/dbfUtils.py

@@ -0,0 +1,121 @@
+import struct, datetime, decimal, itertools
+
+def dbfreader(f):
+    """Returns an iterator over records in a Xbase DBF file.
+
+    The first row returned contains the field names.
+    The second row contains field specs: (type, size, decimal places).
+    Subsequent rows contain the data records.
+    If a record is marked as deleted, it is skipped.
+
+    File should be opened for binary reads.
+
+    """
+    # See DBF format spec at:
+    #     http://www.pgts.com.au/download/public/xbase.htm#DBF_STRUCT
+
+    numrec, lenheader = struct.unpack('<xxxxLH22x', f.read(32))    
+    numfields = (lenheader - 33) // 32
+
+    fields = []
+    for fieldno in xrange(numfields):
+        name, typ, size, deci = struct.unpack('<11sc4xBB14x', f.read(32))
+        name = name.replace('\0', '')       # eliminate NULs from string   
+        fields.append((name, typ, size, deci))
+    yield [field[0] for field in fields]
+    yield [tuple(field[1:]) for field in fields]
+
+    terminator = f.read(1)
+    assert terminator == '\r'
+
+    fields.insert(0, ('DeletionFlag', 'C', 1, 0))
+    fmt = ''.join(['%ds' % fieldinfo[2] for fieldinfo in fields])
+    fmtsiz = struct.calcsize(fmt)
+    for i in xrange(numrec):
+        record = struct.unpack(fmt, f.read(fmtsiz))
+        if record[0] != ' ':
+            continue                        # deleted record
+        result = []
+        for (name, typ, size, deci), value in itertools.izip(fields, record):
+            if name == 'DeletionFlag':
+                continue
+            if typ == "N":
+                value = value.replace('\0', '').lstrip()
+                if value == '':
+                    value = 0
+                elif deci:
+                    try:
+                       value = decimal.Decimal(value)
+                    except decimal.InvalidOperation:
+                       # Could be '************************' (for NULL)
+                       value = 0
+                else:
+                    value = value
+            elif typ == 'D':
+                y, m, d = int(value[:4]), int(value[4:6]), int(value[6:8])
+                try:
+                    value = datetime.date(y, m, d)
+                except ValueError:
+                       # Could be '00000000' (for NULL)
+                       value = 0
+            elif typ == 'L':
+                value = (value in 'YyTt' and 'T') or (value in 'NnFf' and 'F') or '?'
+            result.append(value)
+        yield result
+
+
+def dbfwriter(f, fieldnames, fieldspecs, records):
+    """ Return a string suitable for writing directly to a binary dbf file.
+
+    File f should be open for writing in a binary mode.
+
+    Fieldnames should be no longer than ten characters and not include \x00.
+    Fieldspecs are in the form (type, size, deci) where
+        type is one of:
+            C for ascii character data
+            M for ascii character memo data (real memo fields not supported)
+            D for datetime objects
+            N for ints or decimal objects
+            L for logical values 'T', 'F', or '?'
+        size is the field width
+        deci is the number of decimal places in the provided decimal object
+    Records can be an iterable over the records (sequences of field values).
+    
+    """
+    # header info
+    ver = 3
+    now = datetime.datetime.now()
+    yr, mon, day = now.year-1900, now.month, now.day
+    numrec = len(records)
+    numfields = len(fieldspecs)
+    lenheader = numfields * 32 + 33
+    lenrecord = sum(field[1] for field in fieldspecs) + 1
+    hdr = struct.pack('<BBBBLHH20x', ver, yr, mon, day, numrec, lenheader, lenrecord)
+    f.write(hdr)
+                      
+    # field specs
+    for name, (typ, size, deci) in itertools.izip(fieldnames, fieldspecs):
+        name = name.ljust(11, '\x00')
+        fld = struct.pack('<11sc4xBB14x', name, typ, size, deci)
+        f.write(fld)
+
+    # terminator
+    f.write('\r')
+
+    # records
+    for record in records:
+        f.write(' ')                        # deletion flag
+        for (typ, size, deci), value in itertools.izip(fieldspecs, record):
+            if typ == "N":
+                value = str(value).rjust(size, ' ')
+            elif typ == 'D':
+                value = value.strftime('%Y%m%d')
+            elif typ == 'L':
+                value = str(value)[0].upper()
+            else:
+                value = str(value)[:size].ljust(size, ' ')
+            assert len(value) == size
+            f.write(value)
+
+    # End of file
+    f.write('\x1A')

lib/importer/misc/shp_normalizer.py

@@ -0,0 +1,111 @@
+from chardet.universaldetector import UniversalDetector
+import itertools
+import os.path
+import sys
+import dbfUtils
+import sys
+from osgeo import osr
+from urllib import urlencode
+from urllib2 import urlopen
+import json
+import subprocess
+
+if len(sys.argv) != 3:
+    print "usage: python %s shp_file name" % sys.argv[0]
+    sys.exit()
+shp_file = sys.argv[1]
+name = sys.argv[2]
+
+dbf_file = shp_file[0:-4] + '.dbf'
+prj_file = shp_file[0:-4] + '.prj'
+
+shape_name = os.path.splitext(shp_file)[0]
+
+def get_spatial_reference(shapefile):
+    srs = osr.SpatialReference()
+    shpfile = os.path.basename(shape_name)
+    ret = srs.SetFromUserInput(prj_string)
+    proj4 = srs.ExportToProj4()
+    if not proj4:
+        prj_file = open(shape_name +'.prj','r')
+        prj_lines = prj_file.readlines()
+        prj_file.close()
+        for i in range(len(prj_lines)):
+            prj_lines[i] = prj_string.rstrip( prj_lines[i] )
+        srs = osr.SpatialReference()
+        srs.ImportFromESRI(prj_lines)
+        proj4 = srs.ExportToProj4()
+        """
+        if not proj4:
+            #print 'Failed to convert prj of %s, giving up...' % shpfile
+        else:
+            #print 'Second try assuming ESRI wkt worked for %s!' % shpfile
+        """
+        srs.from_esri = True
+    else:
+        srs.from_esri = False
+    srs.AutoIdentifyEPSG()
+    return srs
+
+def to_epsg(srs):
+    if srs.IsGeographic():
+        return srs.GetAuthorityCode('GEOGCS')
+    else:
+        c = srs.GetAuthorityCode('PROJCS')
+        if c:
+            return c
+        else:
+            try:
+                return srs.GetAuthorityCode('GEOGCS')
+            except:
+                return None
+
+
+srid = None
+
+#Try detecting the SRID
+if os.path.isfile(prj_file):
+  prj_string = open(prj_file,'r').read()
+  code = to_epsg(get_spatial_reference(shp_file))
+
+  srid = code if code else None
+
+try:
+# Try to detect the encoding
+    dbf = open(dbf_file.strip(), 'rb')
+    db = dbfUtils.dbfreader(dbf)
+
+    fnames = db.next()
+    ftypes = db.next()
+
+    # find string fields
+    sfields = []
+    for fno in range(len(fnames)):
+      if ( ftypes[fno][0] == 'C' ) : sfields.append(fno)
+
+    detector = UniversalDetector()
+
+    # TODO: Make this a % of total table size and stop guessing correct values for guessing
+    for row in itertools.islice(db, 1000):
+      # Feed detector with concatenated string fields
+      detector.feed( ''.join(row[fno] for fno in sfields) )
+      if detector.done: break
+    dbf.close()
+    detector.close()
+    encoding = detector.result["encoding"]
+    confidence = detector.result["confidence"]
+    if encoding=="ascii":
+        encoding="LATIN1" # why not UTF8 here ?
+    # There's problems detecting LATIN1 encodings, it detects KOI8-R instead of LATIN1
+    if encoding=="KOI8-R":
+        encoding="LATIN1"
+    # Fix for #1336: since ISO-8859-2 is unlikely and UniversalDetector doesn't support ISO-8859-1, 
+    # we'll fallback to ISO-8859-1 if confidence is not high
+    if encoding=="ISO-8859-2" and confidence < 0.75:
+        encoding="ISO-8859-1"
+except Exception as err:
+    encoding="None" # why not UTF8 here ?
+    #sys.stderr.write(repr(err)+'\n')
+    #sys.exit(1)
+
+print "%s,%s,%s,%s" % (srid,encoding,shp_file,name)

lib/importer/misc/srid_from_gdal.py

@@ -0,0 +1,11 @@
+import sys
+from osgeo import gdal, osr
+
+f = sys.argv[1]
+g = gdal.Open(f)
+proj = g.GetProjection()
+srs = osr.SpatialReference()
+srs.ImportFromESRI([proj])
+srs.AutoIdentifyEPSG()
+srid = srs.GetAuthorityCode(None)
+print "%s" %(srid)