From a11796ef75d0be1b69ce87a8442f9b50b7ed7e40 Mon Sep 17 00:00:00 2001 From: nzkarit Date: Mon, 21 May 2018 18:44:27 +1200 Subject: [PATCH 1/7] Ignore CSV files --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 5fdf849..5788fc4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ *.iq8s *.pyc *.log* +*.csv # Don't include the generated CSV related files generated/ From 6fd3bf4040ab003887e467a4bf481597dac4c653 Mon Sep 17 00:00:00 2001 From: nzkarit Date: Mon, 21 May 2018 18:59:02 +1200 Subject: [PATCH 2/7] Keep the example csv --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 5788fc4..3452c1d 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,9 @@ *.log* *.csv +# Keep the example CSV +!example.csv + # Don't include the generated CSV related files generated/ allICAO.py From 7f70993fdd1bb406a3a0fe607bd7e2162435abbf Mon Sep 17 00:00:00 2001 From: nzkarit Date: Mon, 21 May 2018 19:45:08 +1200 Subject: [PATCH 3/7] WIP for fr24 csv --- FR24csv.py | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 FR24csv.py diff --git a/FR24csv.py b/FR24csv.py new file mode 100644 index 0000000..0856f45 --- /dev/null +++ b/FR24csv.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +# +# Will take a FR24 CSV and make a ADSB_Encoder CSV +import csv +import os +import argparse +import configparser + +def auto_int(x): + """Parses HEX into for argParser""" + return int(x, 0) + +def argParser(): + description = 'This script will take a FR24 CSV file and convert it into a format for ADSB_Encoder.py' + parser = argparse.ArgumentParser(description=description) + parser.add_argument('-i', '--icao', action='store', type=auto_int, dest='icao', default=cfg.get('plane', 'icao'), help='The ICAO number for the plane in hex. Ensure the ICAO is prefixed with \'0x\' to ensure this is parsed as a hex number. This is 24 bits long. Default: %(default)s') + parser.add_argument('--csv', '--csvfile', '--in', '--input', action='store', type=str, dest='csvfile', help='The name of the FR24 CSV file', required=True) + return parser.parse_args() + +def main(): + global cfg + cfg = configparser.ConfigParser() + cfg.read('config.cfg') + + arguments = argParser() + + csvFilename = 'fr24.csv' + + time = 0 + + with open(csvFilename, 'w', newline='') as csvfileout: + output = csv.writer(csvfileout, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) + output.writerow(['time', 'icao', 'latitude', 'longitude', 'altitude']) + with open(arguments.csvfile, newline='') as csvfilein: + reader = csv.DictReader(csvfilein, delimiter=',') + for row in reader: + if time == 0: + time = int(row['Timestamp']) + rowtime = int(row['Timestamp']) - time + newrow = [rowtime, hex(arguments.icao), 'lat', 'long', row['Altitude']] + output.writerow([newrow]) + print(newrow) + csvfileout.close() + + +if __name__ == "__main__": + main() \ No newline at end of file From a8cf3da7452c60ff2984ddfcc9d1d8dee1dcfe11 Mon Sep 17 00:00:00 2001 From: nzkarit Date: Wed, 23 May 2018 17:14:17 +1200 Subject: [PATCH 4/7] FR24 CSV import support complete --- ADSB_Encoder.py | 3 +++ FR24csv.py | 34 ++++++++++++++++++++++------------ 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/ADSB_Encoder.py b/ADSB_Encoder.py index 637f295..3bacd2e 100755 --- a/ADSB_Encoder.py +++ b/ADSB_Encoder.py @@ -105,8 +105,10 @@ def manyPlanes(arguments): row['longitude'] = float(row['longitude']) if not 'altitude' in row.keys(): row['altitude'] = arguments.altitude + print('here') else: row['altitude'] = float(row['altitude']) + print('there') if not 'capability' in row.keys(): row['capability'] = arguments.capability if not 'typecode' in row.keys(): @@ -121,6 +123,7 @@ def manyPlanes(arguments): row['surface'] = arguments.surface logger.debug('Row from CSV: %s' % (row)) modes = ModeS() + print(row['altitude']) (df17_even, df17_odd) = modes.df17_pos_rep_encode(row['capability'], row['icao'], row['typecode'], row['surveillancestatus'], row['nicsupplementb'], row['altitude'], row['time'], row['latitude'], row['longitude'], row['surface']) ppm = PPM() diff --git a/FR24csv.py b/FR24csv.py index 0856f45..d4bb15a 100644 --- a/FR24csv.py +++ b/FR24csv.py @@ -17,6 +17,16 @@ def argParser(): parser.add_argument('--csv', '--csvfile', '--in', '--input', action='store', type=str, dest='csvfile', help='The name of the FR24 CSV file', required=True) return parser.parse_args() +def reverseCSV(csvfile): + """Reverse a CSV. Returns a dictionary of the CSV""" + data = [] + with open(csvfile, newline='') as csvfilein: + reader = csv.DictReader(csvfilein, delimiter=',') + for row in reader: + data.append(row) + csvfilein.close() + return reversed(data) + def main(): global cfg cfg = configparser.ConfigParser() @@ -27,19 +37,19 @@ def main(): csvFilename = 'fr24.csv' time = 0 - + #Need to reverse the FR24 CSV as it is in reverse order i.e. the most recent record is row 2 and the first ADS-B message of the flight is the last row in the CSV + data = reverseCSV(arguments.csvfile) with open(csvFilename, 'w', newline='') as csvfileout: - output = csv.writer(csvfileout, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) - output.writerow(['time', 'icao', 'latitude', 'longitude', 'altitude']) - with open(arguments.csvfile, newline='') as csvfilein: - reader = csv.DictReader(csvfilein, delimiter=',') - for row in reader: - if time == 0: - time = int(row['Timestamp']) - rowtime = int(row['Timestamp']) - time - newrow = [rowtime, hex(arguments.icao), 'lat', 'long', row['Altitude']] - output.writerow([newrow]) - print(newrow) + fieldnames = ['timestamp', 'icao', 'latitude', 'longitude', 'altitude'] + output = csv.DictWriter(csvfileout, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL, fieldnames=fieldnames) + output.writeheader() + for row in data: + if time == 0: + time = int(row['Timestamp']) + rowtime = int(row['Timestamp']) - time + position = row['Position'].split(',') + newrow = {'timestamp':rowtime, 'icao':hex(arguments.icao), 'latitude':position[0], 'longitude':position[1], 'altitude':row['Altitude']} + output.writerow(newrow) csvfileout.close() From 6a9cf210bb2b9a478f118333c7c996a3551e6e61 Mon Sep 17 00:00:00 2001 From: nzkarit Date: Wed, 23 May 2018 17:21:52 +1200 Subject: [PATCH 5/7] README update and fixes while writing readme --- ADSB_Encoder.py | 3 --- FR24csv.py | 2 +- README.md | 20 ++++++++++++++++++++ 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/ADSB_Encoder.py b/ADSB_Encoder.py index 3bacd2e..637f295 100755 --- a/ADSB_Encoder.py +++ b/ADSB_Encoder.py @@ -105,10 +105,8 @@ def manyPlanes(arguments): row['longitude'] = float(row['longitude']) if not 'altitude' in row.keys(): row['altitude'] = arguments.altitude - print('here') else: row['altitude'] = float(row['altitude']) - print('there') if not 'capability' in row.keys(): row['capability'] = arguments.capability if not 'typecode' in row.keys(): @@ -123,7 +121,6 @@ def manyPlanes(arguments): row['surface'] = arguments.surface logger.debug('Row from CSV: %s' % (row)) modes = ModeS() - print(row['altitude']) (df17_even, df17_odd) = modes.df17_pos_rep_encode(row['capability'], row['icao'], row['typecode'], row['surveillancestatus'], row['nicsupplementb'], row['altitude'], row['time'], row['latitude'], row['longitude'], row['surface']) ppm = PPM() diff --git a/FR24csv.py b/FR24csv.py index d4bb15a..dfcbd58 100644 --- a/FR24csv.py +++ b/FR24csv.py @@ -11,7 +11,7 @@ def auto_int(x): return int(x, 0) def argParser(): - description = 'This script will take a FR24 CSV file and convert it into a format for ADSB_Encoder.py' + description = 'This script will take a FR24 CSV file and convert it into a format for FR24csv.py' parser = argparse.ArgumentParser(description=description) parser.add_argument('-i', '--icao', action='store', type=auto_int, dest='icao', default=cfg.get('plane', 'icao'), help='The ICAO number for the plane in hex. Ensure the ICAO is prefixed with \'0x\' to ensure this is parsed as a hex number. This is 24 bits long. Default: %(default)s') parser.add_argument('--csv', '--csvfile', '--in', '--input', action='store', type=str, dest='csvfile', help='The name of the FR24 CSV file', required=True) diff --git a/README.md b/README.md index 77bcf85..7319d5f 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,26 @@ These CSV files can be used for input into the application ## generateAllICAO.py This script will generate a CSV with all the different ICAO numbers in it. +# Import FlightRadar24 CSV files +This script will take a FR24 CSV files and convert it ready for import into ADSB_Encoder.py. It outputs a file called fr24.csv. +``` +$ ./FR24csv.py --csv +$ ./ADSB_Encoder.py --csv fr24.csv + +$ ./FR24csv.py --help +usage: FR24csv.py [-h] [-i ICAO] --csv CSVFILE + +This script will take a FR24 CSV file and convert it into a format for +FR24csv.py + +optional arguments: + -h, --help show this help message and exit + -i ICAO, --icao ICAO The ICAO number for the plane in hex. Ensure the ICAO + is prefixed with '0x' to ensure this is parsed as a + hex number. This is 24 bits long. Default: 0x75008F + --csv CSVFILE, --csvfile CSVFILE, --in CSVFILE, --input CSVFILE + The name of the FR24 CSV file +``` # References 1. "*Gr-Air-Modes*", **Nick Foster**, 2012 1. "*EXPLOITING THE AUTOMATIC DEPENDENT SURVEILLANCE BROADCAST SYSTEM VIA FALSE TARGET INJECTION*", **Domenic Magazu III**, 2012 From 46a3ca9dd261c2348fbfcf527104619b1320fabb Mon Sep 17 00:00:00 2001 From: nzkarit Date: Wed, 23 May 2018 22:53:38 +1200 Subject: [PATCH 6/7] Adding a pause between message pairs. Currently set to make one pair a second --- ADSB_Encoder.py | 25 +++++------ FR24csv.py | 112 ++++++++++++++++++++++++------------------------ PPM.py | 13 ++++++ config.cfg | 4 +- 4 files changed, 85 insertions(+), 69 deletions(-) mode change 100644 => 100755 FR24csv.py diff --git a/ADSB_Encoder.py b/ADSB_Encoder.py index 637f295..b7afacb 100755 --- a/ADSB_Encoder.py +++ b/ADSB_Encoder.py @@ -60,11 +60,9 @@ def argParser(): parser.add_argument('-s', '--surface', action='store', default=cfg.getboolean('plane', 'surface'), type=auto_bool, dest='surface', help='If the plane is on the ground or not. Default: %(default)s') parser.add_argument('-o', '--out', '--output', action='store', type=str, default=cfg.get('general', 'outputfilename'), dest='outputfilename', help='The iq8s output filename. This is the file which you will feed into the hackRF. Default: %(default)s') parser.add_argument('-r', '--repeats', action='store', dest='repeats', type=int, default=cfg.getint('general', 'repeats'), help='How many repeats of the data to perform. Default: %(default)s') - parser.add_argument('--csv', '--csvfile', '--in', '--input', action='store', type=str, default=cfg.get('general', 'csvfile'), dest='csvfile', help='Import a CSV file with the plane data in it. Default: %(default)s') + parser.add_argument('--csv', '--csvfile', '--in', '--input', action='store', type=str, default=cfg.get('general', 'csvfile'), dest='csvfile', help='Import a CSV file with the plane data in it. Default: %(default)s') + parser.add_argument('--intermessagegap', action='store', type=int, default=cfg.get('general', 'intermessagegap'), dest='intermessagegap', help='When repeating or reading a CSV the number of microseconds between messages. Default: %(default)s') # TODO Make it so it can do a static checksum - # TODO Get pause between messages - # TODO Get pause between repeats - # TODO Do a pause function return parser.parse_args() def singlePlane(arguments): @@ -81,16 +79,19 @@ def singlePlane(arguments): hackrf = HackRF() samples_array = hackrf.hackrf_raw_IQ_format(df17_array) samples = samples+samples_array + gap_array = ppm.addGap(arguments.intermessagegap) + samples_array = hackrf.hackrf_raw_IQ_format(gap_array) + samples = samples+samples_array return samples def manyPlanes(arguments): - logger.info('Processing CSV file: %s' % (arguments.csvfile)) + logger.info('Processing CSV file: %s' % (arguments.csvfile)) samples = bytearray() logger.info('Repeating the message %s times' % (arguments.repeats)) for i in range(0, arguments.repeats): with open(arguments.csvfile, newline='') as csvfile: reader = csv.DictReader(csvfile, delimiter=',') - for row in reader: + for row in reader: if not 'icao' in row.keys(): row['icao'] = arguments.icao else: @@ -129,6 +130,9 @@ def manyPlanes(arguments): hackrf = HackRF() samples_array = hackrf.hackrf_raw_IQ_format(df17_array) samples = samples+samples_array + gap_array = ppm.addGap(arguments.intermessagegap) + samples_array = hackrf.hackrf_raw_IQ_format(gap_array) + samples = samples+samples_array return samples def writeOutputFile(filename, data): @@ -138,11 +142,11 @@ def writeOutputFile(filename, data): SamplesFile.write(data) SamplesFile.close() os.system('sync') - os.system('rm %s' % (filename)) + os.system('rm %s' % (filename)) logger.info('dd for file: %s' % (filename)) - os.system("dd if=%s of=%s bs=4k seek=63 > /dev/null 2>&1" % (tmpfile, filename)) + os.system("dd if=%s of=%s bs=4k seek=63 > /dev/null 2>&1" % (tmpfile, filename)) os.system('sync') - os.system('rm %s'%(tmpfile)) + os.system('rm %s'%(tmpfile)) def main(): global cfg @@ -188,6 +192,3 @@ def threadingCSV(csv): if __name__ == "__main__": main() - - - diff --git a/FR24csv.py b/FR24csv.py old mode 100644 new mode 100755 index dfcbd58..a7ab15d --- a/FR24csv.py +++ b/FR24csv.py @@ -1,57 +1,57 @@ -#!/usr/bin/env python3 -# -# Will take a FR24 CSV and make a ADSB_Encoder CSV -import csv -import os -import argparse -import configparser - -def auto_int(x): - """Parses HEX into for argParser""" - return int(x, 0) - -def argParser(): - description = 'This script will take a FR24 CSV file and convert it into a format for FR24csv.py' - parser = argparse.ArgumentParser(description=description) - parser.add_argument('-i', '--icao', action='store', type=auto_int, dest='icao', default=cfg.get('plane', 'icao'), help='The ICAO number for the plane in hex. Ensure the ICAO is prefixed with \'0x\' to ensure this is parsed as a hex number. This is 24 bits long. Default: %(default)s') - parser.add_argument('--csv', '--csvfile', '--in', '--input', action='store', type=str, dest='csvfile', help='The name of the FR24 CSV file', required=True) - return parser.parse_args() - -def reverseCSV(csvfile): - """Reverse a CSV. Returns a dictionary of the CSV""" - data = [] - with open(csvfile, newline='') as csvfilein: - reader = csv.DictReader(csvfilein, delimiter=',') - for row in reader: - data.append(row) - csvfilein.close() - return reversed(data) - -def main(): - global cfg - cfg = configparser.ConfigParser() - cfg.read('config.cfg') - - arguments = argParser() - - csvFilename = 'fr24.csv' - - time = 0 - #Need to reverse the FR24 CSV as it is in reverse order i.e. the most recent record is row 2 and the first ADS-B message of the flight is the last row in the CSV - data = reverseCSV(arguments.csvfile) - with open(csvFilename, 'w', newline='') as csvfileout: - fieldnames = ['timestamp', 'icao', 'latitude', 'longitude', 'altitude'] - output = csv.DictWriter(csvfileout, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL, fieldnames=fieldnames) - output.writeheader() - for row in data: - if time == 0: - time = int(row['Timestamp']) - rowtime = int(row['Timestamp']) - time - position = row['Position'].split(',') - newrow = {'timestamp':rowtime, 'icao':hex(arguments.icao), 'latitude':position[0], 'longitude':position[1], 'altitude':row['Altitude']} - output.writerow(newrow) - csvfileout.close() - - -if __name__ == "__main__": +#!/usr/bin/env python3 +# +# Will take a FR24 CSV and make a ADSB_Encoder CSV +import csv +import os +import argparse +import configparser + +def auto_int(x): + """Parses HEX into for argParser""" + return int(x, 0) + +def argParser(): + description = 'This script will take a FR24 CSV file and convert it into a format for FR24csv.py' + parser = argparse.ArgumentParser(description=description) + parser.add_argument('-i', '--icao', action='store', type=auto_int, dest='icao', default=cfg.get('plane', 'icao'), help='The ICAO number for the plane in hex. Ensure the ICAO is prefixed with \'0x\' to ensure this is parsed as a hex number. This is 24 bits long. Default: %(default)s') + parser.add_argument('--csv', '--csvfile', '--in', '--input', action='store', type=str, dest='csvfile', help='The name of the FR24 CSV file', required=True) + return parser.parse_args() + +def reverseCSV(csvfile): + """Reverse a CSV. Returns a dictionary of the CSV""" + data = [] + with open(csvfile, newline='') as csvfilein: + reader = csv.DictReader(csvfilein, delimiter=',') + for row in reader: + data.append(row) + csvfilein.close() + return reversed(data) + +def main(): + global cfg + cfg = configparser.ConfigParser() + cfg.read('config.cfg') + + arguments = argParser() + + csvFilename = 'fr24.csv' + + time = 0 + #Need to reverse the FR24 CSV as it is in reverse order i.e. the most recent record is row 2 and the first ADS-B message of the flight is the last row in the CSV + data = reverseCSV(arguments.csvfile) + with open(csvFilename, 'w', newline='') as csvfileout: + fieldnames = ['timestamp', 'icao', 'latitude', 'longitude', 'altitude'] + output = csv.DictWriter(csvfileout, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL, fieldnames=fieldnames) + output.writeheader() + for row in data: + if time == 0: + time = int(row['Timestamp']) + rowtime = int(row['Timestamp']) - time + position = row['Position'].split(',') + newrow = {'timestamp':rowtime, 'icao':hex(arguments.icao), 'latitude':position[0], 'longitude':position[1], 'altitude':row['Altitude']} + output.writerow(newrow) + csvfileout.close() + + +if __name__ == "__main__": main() \ No newline at end of file diff --git a/PPM.py b/PPM.py index f3eaad2..fdd05a8 100644 --- a/PPM.py +++ b/PPM.py @@ -46,3 +46,16 @@ class PPM: #print '[{}]'.format(', '.join(hex(x) for x in ppm)) return bytearray(ppm) + + def addGap(self, gap): + """ + This function will add dead air as a gap between messages + Args: + gap: The number of microseconds to have as a gap + Returns: + The bytearray of the PPM data + """ + ppm = [ ] + for i in range(gap): # pause + ppm.append( 0 ) + return bytearray(ppm) diff --git a/config.cfg b/config.cfg index 2b16237..9d3b9b9 100644 --- a/config.cfg +++ b/config.cfg @@ -1,7 +1,9 @@ [general] outputfilename = Samples_256K.iq8s repeats = 1 -csvfile = +csvfile = +# Currently in PPM.py one message is 48 dead air, 8 preamble, 112 message, 100 dead air, 8 preamble, 112 message, 48 dead air leaves us with 99564 microseconds to make a second +intermessagegap = 99564 [plane] icao = 0x75008F From 59fdee124c6c543173c15f86cf3b8770b51b60bd Mon Sep 17 00:00:00 2001 From: nzkarit Date: Thu, 24 May 2018 21:13:55 +1200 Subject: [PATCH 7/7] Added an option to run Flight Radar 24 CSVs in realtime --- ADSB_Encoder.py | 10 +++++++++- config.cfg | 3 +++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/ADSB_Encoder.py b/ADSB_Encoder.py index b7afacb..ce7b053 100755 --- a/ADSB_Encoder.py +++ b/ADSB_Encoder.py @@ -62,6 +62,7 @@ def argParser(): parser.add_argument('-r', '--repeats', action='store', dest='repeats', type=int, default=cfg.getint('general', 'repeats'), help='How many repeats of the data to perform. Default: %(default)s') parser.add_argument('--csv', '--csvfile', '--in', '--input', action='store', type=str, default=cfg.get('general', 'csvfile'), dest='csvfile', help='Import a CSV file with the plane data in it. Default: %(default)s') parser.add_argument('--intermessagegap', action='store', type=int, default=cfg.get('general', 'intermessagegap'), dest='intermessagegap', help='When repeating or reading a CSV the number of microseconds between messages. Default: %(default)s') + parser.add_argument('--realtime', action='store', default=cfg.getboolean('general', 'realtime'), type=auto_bool, dest='realtime', help='When running a CSV which has a timestamp column whether to run in realtime following the timestamp or if just follow intermessagegap. If realtime is set it will override intermessagegap. Default: %(default)s') # TODO Make it so it can do a static checksum return parser.parse_args() @@ -88,10 +89,12 @@ def manyPlanes(arguments): logger.info('Processing CSV file: %s' % (arguments.csvfile)) samples = bytearray() logger.info('Repeating the message %s times' % (arguments.repeats)) + prevtimestamp = 0 for i in range(0, arguments.repeats): with open(arguments.csvfile, newline='') as csvfile: reader = csv.DictReader(csvfile, delimiter=',') for row in reader: + gap = arguments.intermessagegap if not 'icao' in row.keys(): row['icao'] = arguments.icao else: @@ -120,6 +123,11 @@ def manyPlanes(arguments): row['time'] = arguments.time if not 'surface' in row.keys(): row['surface'] = arguments.surface + if 'timestamp' in row.keys(): + if arguments.realtime: + gap = int(row['timestamp']) - prevtimestamp + gap = gap * 100000 + prevtimestamp = int(row['timestamp']) logger.debug('Row from CSV: %s' % (row)) modes = ModeS() (df17_even, df17_odd) = modes.df17_pos_rep_encode(row['capability'], row['icao'], row['typecode'], row['surveillancestatus'], row['nicsupplementb'], row['altitude'], row['time'], row['latitude'], row['longitude'], row['surface']) @@ -130,7 +138,7 @@ def manyPlanes(arguments): hackrf = HackRF() samples_array = hackrf.hackrf_raw_IQ_format(df17_array) samples = samples+samples_array - gap_array = ppm.addGap(arguments.intermessagegap) + gap_array = ppm.addGap(gap) samples_array = hackrf.hackrf_raw_IQ_format(gap_array) samples = samples+samples_array return samples diff --git a/config.cfg b/config.cfg index 9d3b9b9..d9fdce0 100644 --- a/config.cfg +++ b/config.cfg @@ -4,6 +4,9 @@ repeats = 1 csvfile = # Currently in PPM.py one message is 48 dead air, 8 preamble, 112 message, 100 dead air, 8 preamble, 112 message, 48 dead air leaves us with 99564 microseconds to make a second intermessagegap = 99564 +# This is the message length from PPM.py +messagelength = 436 +realtime = false [plane] icao = 0x75008F