trackdirect2/server/trackdirect/parser/AprsPacketParser.py

630 lines
28 KiB
Python

import logging
from twisted.python import log
import json
import datetime
import time
import calendar
import hashlib
from trackdirect.exceptions.TrackDirectParseError import TrackDirectParseError
from trackdirect.exceptions.TrackDirectMissingSenderError import TrackDirectMissingSenderError
from trackdirect.exceptions.TrackDirectMissingStationError import TrackDirectMissingStationError
from trackdirect.repositories.OgnHiddenStationRepository import OgnHiddenStationRepository
from trackdirect.repositories.OgnDeviceRepository import OgnDeviceRepository
from trackdirect.repositories.StationRepository import StationRepository
from trackdirect.repositories.SenderRepository import SenderRepository
from trackdirect.repositories.PacketRepository import PacketRepository
from trackdirect.repositories.PacketTelemetryRepository import PacketTelemetryRepository
from trackdirect.repositories.PacketWeatherRepository import PacketWeatherRepository
from trackdirect.repositories.PacketOgnRepository import PacketOgnRepository
from trackdirect.repositories.MarkerRepository import MarkerRepository
from trackdirect.repositories.StationTelemetryBitsRepository import StationTelemetryBitsRepository
from trackdirect.repositories.StationTelemetryEqnsRepository import StationTelemetryEqnsRepository
from trackdirect.repositories.StationTelemetryParamRepository import StationTelemetryParamRepository
from trackdirect.repositories.StationTelemetryUnitRepository import StationTelemetryUnitRepository
from trackdirect.objects.Packet import Packet
from trackdirect.objects.Station import Station
from trackdirect.objects.PacketWeather import PacketWeather
from trackdirect.objects.PacketTelemetry import PacketTelemetry
from trackdirect.objects.Marker import Marker
from trackdirect.objects.OgnHiddenStation import OgnHiddenStation
from trackdirect.objects.OgnDevice import OgnDevice
from trackdirect.parser.policies.AprsPacketTypePolicy import AprsPacketTypePolicy
from trackdirect.parser.policies.PacketAssumedMoveTypePolicy import PacketAssumedMoveTypePolicy
from trackdirect.parser.policies.PacketSpeedComputablePolicy import PacketSpeedComputablePolicy
from trackdirect.parser.policies.PreviousPacketPolicy import PreviousPacketPolicy
from trackdirect.parser.policies.PacketTailPolicy import PacketTailPolicy
from trackdirect.parser.policies.PacketRelatedMapSectorsPolicy import PacketRelatedMapSectorsPolicy
from trackdirect.parser.policies.PacketMapIdPolicy import PacketMapIdPolicy
from trackdirect.parser.policies.PacketPathPolicy import PacketPathPolicy
from trackdirect.parser.policies.MapSectorPolicy import MapSectorPolicy
from trackdirect.parser.policies.PacketCommentPolicy import PacketCommentPolicy
from trackdirect.parser.policies.PacketKillCharPolicy import PacketKillCharPolicy
from trackdirect.parser.policies.StationNameFormatPolicy import StationNameFormatPolicy
from trackdirect.parser.policies.PacketOgnDataPolicy import PacketOgnDataPolicy
class AprsPacketParser():
"""AprsPacketParser tackes a aprslib output and converts it to a Trackdirect Packet
"""
def __init__(self, db, saveOgnStationsWithMissingIdentity):
"""The __init__ method.
Args:
db (psycopg2.Connection): Database connection
saveOgnStationsWithMissingIdentity (boolean): True if we should not ignore stationss with a missing identity
"""
self.db = db
self.saveOgnStationsWithMissingIdentity = saveOgnStationsWithMissingIdentity
self.logger = logging.getLogger('trackdirect')
self.databaseWriteAccess = True
self.sourceId = 1
self.ognDeviceRepository = OgnDeviceRepository(db)
self.ognHiddenStationRepository = OgnHiddenStationRepository(db)
self.stationRepository = StationRepository(db)
self.senderRepository = SenderRepository(db)
self.packetRepository = PacketRepository(db)
self.packetTelemetryRepository = PacketTelemetryRepository(db)
self.packetWeatherRepository = PacketWeatherRepository(db)
self.packetOgnRepository = PacketOgnRepository(db)
self.markerRepository = MarkerRepository(db)
self.stationTelemetryBitsRepository = StationTelemetryBitsRepository(db)
self.stationTelemetryEqnsRepository = StationTelemetryEqnsRepository(db)
self.stationTelemetryParamRepository = StationTelemetryParamRepository(db)
self.stationTelemetryUnitRepository = StationTelemetryUnitRepository(db)
def setDatabaseWriteAccess(self, databaseWriteAccess):
"""Enable or disable database updates
Args:
databaseWriteAccess (boolean): True if we have database access otherwise false
"""
self.databaseWriteAccess = databaseWriteAccess
def setSourceId(self, sourceId):
"""Set what source packet is from (APRS, CWOP ...)
Args:
sourceId (int): Id that corresponds to id in source-table
"""
self.sourceId = sourceId
def getPacket(self, data, timestamp=None, minimal=False):
"""Returns the resulting packet
Args:
data (dict): Raw packet data
timestamp (int): Packet timestamp
minimal (boolean): Set to true to only perform minimal parsing
Returns:
Packet
"""
self.isHiddenStation = False
self.data = data
self.packet = Packet(self.db)
self.packet.sourceId = self.sourceId
self._parsePacketInitialValues(timestamp)
self._parsePacketReportedTimestamp()
# Parse OGN stuff before station-id, may result in no station to create
self._parsePacketOgn()
if (self.packet.mapId != 15):
self._parsePacketType()
self._parsePacketSender()
self._parsePacketStationName()
self._parsePacketStationId()
self._parsePacketRng()
self._parsePacketPhg()
if (self.packet.stationId is None):
self.packet.mapId = 4
else:
self._parsePacketComment()
self._parsePacketWeather()
self._parsePacketTelemetryDefinition()
self._parsePacketTelemetry()
self._parsePacketPosition()
self._parsePacketPath()
if (not minimal):
previousPacketPolicy = PreviousPacketPolicy(
self.packet, self.db)
previousPacket = previousPacketPolicy.getPreviousPacket()
self._parseAssumedMoveTypeId(previousPacket)
self._parsePacketTail(previousPacket)
self._parsePacketMapId(previousPacket)
self._parsePacketPreviousTimestamps(previousPacket)
self._parsePacketPhgRngTimestamps(previousPacket)
self._parsePacketRelatedMapSectors(previousPacket)
if (self.isHiddenStation):
if (self.packet.ogn is not None):
self.packet.ogn.ognAddressTypeId = None
self.packet.ogn.ognSenderAddress = None
self.packet.comment = None
self.packet.raw = None
return self.packet
def _parsePacketInitialValues(self, timestamp):
"""Set packet initial values
Args:
timestamp (int): Packet timestamp
"""
self.packet.raw = self.data['raw'].replace('\x00', '')
self.packet.symbol = self.data['symbol'] if (
'symbol' in self.data) else None
self.packet.symbolTable = self.data['symbol_table'] if (
'symbol_table' in self.data) else None
self.packet.reportedTimestamp = self.data['timestamp'] if (
'timestamp' in self.data) else None
self.packet.rawPath = self.data['to'] + ',' + ','.join(self.data['path']) if (
'path' in self.data and 'to' in self.data) else None
if (timestamp is not None):
self.packet.timestamp = timestamp
else:
self.packet.timestamp = int(time.time())
self.packet.positionTimestamp = self.packet.timestamp
self.packet.packetTailTimestamp = 0
self.packet.posambiguity = self.data['posambiguity'] if (
'posambiguity' in self.data) else None
self.packet.speed = self.data['speed'] if (
'speed' in self.data) else None
self.packet.course = self.data['course'] if (
'course' in self.data) else None
self.packet.altitude = self.data['altitude'] if (
'altitude' in self.data) else None
self.packet.mapId = 10 # Default map for a packet without a position
self.packet.markerId = 1
self.packet.markerCounter = 1 # We allways start at 1
def _parsePacketReportedTimestamp(self):
"""Set packet reported timestamp
"""
if ('timestamp' in self.data):
reportedTimestampDay = int(datetime.datetime.utcfromtimestamp(
int(self.data['timestamp'])).strftime('%d'))
currentTimestampDay = int(datetime.datetime.utcfromtimestamp(
self.packet.timestamp).strftime('%d'))
if (reportedTimestampDay > currentTimestampDay and self.data['timestamp'] > self.packet.timestamp):
# Day is in the future, the reported timestamp was probably in previous month
prevMonth = int(datetime.datetime.utcfromtimestamp(
int(self.data['timestamp']) - (reportedTimestampDay+1)*24*60*60).strftime('%m'))
prevMonthYear = int(datetime.datetime.utcfromtimestamp(
int(self.data['timestamp']) - (reportedTimestampDay+1)*24*60*60).strftime('%Y'))
numberOfDaysInMonth = int(
calendar.monthrange(prevMonthYear, prevMonth)[1])
self.packet.reportedTimestamp = int(
self.data['timestamp']) - (numberOfDaysInMonth*24*60*60)
else:
self.packet.reportedTimestamp = None
def _parsePacketRng(self):
"""Set packet RNG data
"""
if ('rng' in self.data):
try:
self.packet.rng = float(self.data['rng'])
self.packet.latestRngTimestamp = self.packet.timestamp
if (self.packet.rng <= 0):
self.packet.rng = None
except ValueError:
# Not a float
self.packet.rng = None
def _parsePacketPhg(self):
"""Set packet PHG data
"""
if ('phg' in self.data):
phg = str(self.data['phg'])[0:4]
if (len(phg) == 4 and phg.isdigit()):
self.packet.phg = phg
self.packet.latestPhgTimestamp = self.packet.timestamp
def _parsePacketSender(self):
"""Set sender id and name
"""
stationNameFormatPolicy = StationNameFormatPolicy()
self.packet.senderName = stationNameFormatPolicy.getCorrectFormat(
self.data["from"])
try:
sender = self.senderRepository.getCachedObjectByName(
self.packet.senderName)
self.packet.senderId = sender.id
except (TrackDirectMissingSenderError) as exp:
if (self.databaseWriteAccess):
sender = self.senderRepository.getObjectByName(
self.packet.senderName, True, self.packet.sourceId)
self.packet.senderId = sender.id
def _parsePacketStationName(self):
"""Set station name
"""
if ("object_name" in self.data
and self.data["object_name"] is not None
and self.data["object_name"] != ''):
self.packet.stationTypeId = 2
stationNameFormatPolicy = StationNameFormatPolicy()
self.packet.stationName = stationNameFormatPolicy.getCorrectFormat(
self.data["object_name"])
if (self.packet.stationName is None or self.packet.stationName == ''):
self.packet.stationName = self.packet.senderName
if (self.packet.stationName == self.packet.senderName):
self.packet.stationTypeId = 1
else:
self.packet.stationTypeId = 1
self.packet.stationName = self.packet.senderName
def _parsePacketStationId(self):
"""Set station id
"""
if (self.packet.stationName is None):
return
try:
station = self.stationRepository.getCachedObjectByName(
self.packet.stationName,
self.packet.sourceId
)
self.packet.stationId = station.id
if (self.packet.stationName != station.name):
# Station lower/upper case is modified
station.name = self.packet.stationName
station.save()
if (station.stationTypeId != self.packet.stationTypeId and int(self.packet.stationTypeId) == 1):
# Switch to station type 1
station.stationTypeId = self.packet.stationTypeId
station.save()
except (TrackDirectMissingStationError) as exp:
if (self.databaseWriteAccess):
station = self.stationRepository.getObjectByName(
self.packet.stationName,
self.packet.sourceId,
self.packet.stationTypeId,
True
)
self.packet.stationId = station.id
def _parsePacketComment(self):
"""Set packet comment
"""
commentPolicy = PacketCommentPolicy()
self.packet.comment = commentPolicy.getComment(
self.data, self.packet.packetTypeId)
def _parsePacketOgn(self):
"""Set th OGN sender address
"""
ognDataPolicy = PacketOgnDataPolicy(
self.data, self.ognDeviceRepository, self.packet.sourceId)
if (ognDataPolicy.isOgnPositionPacket):
originalRaw = self.packet.raw
if (not ognDataPolicy.isAllowedToTrack):
self.packet.mapId = 15
return
elif (not ognDataPolicy.isAllowedToIdentify):
if (not self.saveOgnStationsWithMissingIdentity) :
self.packet.mapId = 15
return
self.isHiddenStation = True
self.data["from"] = self._getHiddenStationName()
self.data["object_name"] = None
self.data['ogn'] = ognDataPolicy.getOgnData()
if (self.data['ogn'] is not None):
self.packet.ogn = self.packetOgnRepository.getObjectFromPacketData(
self.data)
self.packet.ogn.stationId = self.packet.stationId
self.packet.ogn.timestamp = self.packet.timestamp
self.data["comment"] = None
self._modifyOgnSymbol(originalRaw)
def _modifyOgnSymbol(self, originalRaw):
"""Sets a better symbol for an OGN station
Args:
originalRaw (string): Non modified raw
"""
if ((self.packet.symbol == '\'' and self.packet.symbolTable == '/') or (self.packet.symbol == '^' and self.packet.symbolTable in ['/', '\\']) or (self.packet.symbol == 'g' and self.packet.symbolTable in ['/'])):
ognDevice = None
if (self.packet.ogn is not None and self.packet.ogn.ognSenderAddress is not None):
ognDevice = self.ognDeviceRepository.getObjectByDeviceId(
self.packet.ogn.ognSenderAddress)
if (ognDevice is not None and ognDevice.isExistingObject() and ognDevice.ddbAircraftType > 0 and ognDevice.ddbAircraftType < 6):
# If another aircraft type exist in device db, use that instead
if (ognDevice.ddbAircraftType == 1): # Gliders/motoGliders
# Glider -> Glider
self.packet.symbol = '^'
self.packet.symbolTable = 'G'
elif (ognDevice.ddbAircraftType == 2): # Planes
# Powered aircraft -> Propeller aircraft
self.packet.symbol = '^'
self.packet.symbolTable = 'P'
elif (ognDevice.ddbAircraftType == 3): # Ultralights
# Small plane
self.packet.symbol = '\''
self.packet.symbolTable = '/'
elif (ognDevice.ddbAircraftType == 4): # Helicopters
# Helicopter
self.packet.symbol = 'X'
self.packet.symbolTable = '/'
elif (ognDevice.ddbAircraftType == 5): # Drones/UAV
# UAV -> Drone
self.packet.symbol = '^'
self.packet.symbolTable = 'D'
elif (self.packet.ogn is not None and self.packet.ogn.ognAircraftTypeId is not None):
if (self.packet.ogn.ognAircraftTypeId == 1):
# Glider -> Glider
self.packet.symbol = '^'
self.packet.symbolTable = 'G'
elif (self.packet.ogn.ognAircraftTypeId == 9):
# Jet aircraft -> Jet
self.packet.symbol = '^'
self.packet.symbolTable = 'J'
elif (self.packet.ogn.ognAircraftTypeId == 8):
# Powered aircraft -> Propeller aircraft
self.packet.symbol = '^'
self.packet.symbolTable = 'P'
elif (self.packet.ogn.ognAircraftTypeId == 13):
# UAV -> Drone
self.packet.symbol = '^'
self.packet.symbolTable = 'D'
elif (self.packet.ogn.ognAircraftTypeId == 7): #paraglider
# map to own symbol 94-76.svg (do not show hangglider symbol 103-1.svg, 'g' = 103)
self.packet.symbol = '^' #94
self.packet.symbolTable = 'L' #76
if ((self.packet.symbol == '\'' and self.packet.symbolTable == '/') or (self.packet.symbol == '^' and self.packet.symbolTable in ['/', '\\'])):
# Current symbol is still "small aircraft" or "large aircraft"
if (originalRaw.startswith('FMT')):
# FMT, Remotely Piloted
self.packet.symbol = '^'
self.packet.symbolTable = 'R'
def _getHiddenStationName(self):
"""Returnes a unidentifiable station name
Returns:
string
"""
date = datetime.datetime.utcfromtimestamp(
self.packet.timestamp).strftime('%Y%m%d')
dailyStationNameHash = hashlib.sha256(
self.data["from"] + date).hexdigest()
ognHiddenStation = self.ognHiddenStationRepository.getObjectByHashedName(
dailyStationNameHash, True)
return ognHiddenStation.getStationName()
def _parsePacketWeather(self):
"""Parse weather data
"""
if ("weather" in self.data):
self.packet.weather = self.packetWeatherRepository.getObjectFromPacketData(
self.data)
self.packet.weather.stationId = self.packet.stationId
self.packet.weather.timestamp = self.packet.timestamp
def _parsePacketTelemetry(self):
"""Parse telemetry data
"""
if ("telemetry" in self.data):
self.packet.telemetry = self.packetTelemetryRepository.getObjectFromPacketData(
self.data)
self.packet.telemetry.stationId = self.packet.stationId
self.packet.telemetry.timestamp = self.packet.timestamp
def _parsePacketTelemetryDefinition(self):
"""Parse telemetry definition
"""
if ("tBITS" in self.data):
self.packet.stationTelemetryBits = self.stationTelemetryBitsRepository.getObjectFromPacketData(
self.data)
self.packet.stationTelemetryBits.stationId = self.packet.stationId
self.packet.stationTelemetryBits.timestamp = self.packet.timestamp
if ("tEQNS" in self.data):
self.packet.stationTelemetryEqns = self.stationTelemetryEqnsRepository.getObjectFromPacketData(
self.data)
self.packet.stationTelemetryEqns.stationId = self.packet.stationId
self.packet.stationTelemetryEqns.timestamp = self.packet.timestamp
if ("tPARM" in self.data):
self.packet.stationTelemetryParam = self.stationTelemetryParamRepository.getObjectFromPacketData(
self.data)
self.packet.stationTelemetryParam.stationId = self.packet.stationId
self.packet.stationTelemetryParam.timestamp = self.packet.timestamp
if ("tUNIT" in self.data):
self.packet.stationTelemetryUnit = self.stationTelemetryUnitRepository.getObjectFromPacketData(
self.data)
self.packet.stationTelemetryUnit.stationId = self.packet.stationId
self.packet.stationTelemetryUnit.timestamp = self.packet.timestamp
def _parsePacketType(self):
"""Set packet type
"""
aprsPacketTypePolicy = AprsPacketTypePolicy()
self.packet.packetTypeId = aprsPacketTypePolicy.getPacketType(
self.packet)
def _parsePacketPosition(self):
"""Parse the packet position and related attributes
"""
if ("latitude" in self.data
and "longitude" in self.data
and self.data["latitude"] is not None
and self.data["longitude"] is not None):
self.packet.latitude = self.data['latitude']
self.packet.longitude = self.data['longitude']
mapSectorPolicy = MapSectorPolicy()
self.packet.mapSector = mapSectorPolicy.getMapSector(
self.data["latitude"], self.data["longitude"])
self.packet.mapId = 1 # Default map for a packet with a position
self.packet.isMoving = 1 # Moving/Stationary, default value is moving
def _parsePacketPath(self):
"""Parse packet path
"""
if ("path" in self.data and isinstance(self.data["path"], list)):
packetPathPolicy = PacketPathPolicy(
self.data['path'], self.packet.sourceId, self.stationRepository, self.senderRepository)
self.packet.stationIdPath = packetPathPolicy.getStationIdPath()
self.packet.stationNamePath = packetPathPolicy.getStationNamePath()
self.packet.stationLocationPath = packetPathPolicy.getStationLocationPath()
def _getPacketRawBody(self):
"""Returns packet raw body as string
Returns:
Packet raw body as string
"""
try:
(rawHeader, rawBody) = self.data["raw"].split(':', 1)
except:
raise TrackDirectParseError(
'Could not split packet into header and body', self.data)
if len(rawBody) == 0:
raise TrackDirectParseError('Packet body is empty', self.data)
return rawBody
def _parsePacketPhgRngTimestamps(self, previousPacket):
"""Set current packet timestamp when latest rng / phg was received
Args:
previousPacket (Packet): Packet object that represents the packet before the current packet
"""
if (previousPacket.isExistingObject()
and self.packet.isPostitionEqual(previousPacket)):
if (self.packet.phg is None
and previousPacket.latestPhgTimestamp is not None
and previousPacket.latestPhgTimestamp > (self.packet.timestamp - (60*60*24))):
self.latestPhgTimestamp = previousPacket.latestPhgTimestamp
if (self.packet.rng is None
and previousPacket.latestRngTimestamp is not None
and previousPacket.latestRngTimestamp > (self.packet.timestamp - (60*60*24))):
self.latestRngTimestamp = previousPacket.latestRngTimestamp
def _parsePacketMapId(self, previousPacket):
"""Set current packet map id and related
Args:
previousPacket (Packet): Packet object that represents the packet before the current packet
"""
mapIdPolicy = PacketMapIdPolicy(self.packet, previousPacket)
killCharPolicy = PacketKillCharPolicy()
if (killCharPolicy.hasKillCharacter(self.data)):
mapIdPolicy.enableHavingKillCharacter()
if (mapIdPolicy.isReplacingPreviousPacket()):
self.packet.replacePacketId = previousPacket.id
self.packet.replacePacketTimestamp = previousPacket.timestamp
if (mapIdPolicy.isConfirmingPreviousPacket()):
self.packet.confirmPacketId = previousPacket.id
self.packet.confirmPacketTimestamp = previousPacket.timestamp
if (mapIdPolicy.isKillingPreviousPacket()):
self.packet.abnormalPacketId = previousPacket.id
self.packet.abnormalPacketTimestamp = previousPacket.timestamp
self.packet.mapId = mapIdPolicy.getMapId()
self.packet.markerId = mapIdPolicy.getMarkerId()
if (self.packet.markerId is None):
# Policy could not find a marker id to use, create a new
self.packet.markerId = self._getNewMarkerId()
if (self.packet.markerId == 1):
self.packet.mapId = 4
def _parsePacketPreviousTimestamps(self, previousPacket):
"""Set packet previous timestamps and related
Args:
previousPacket (Packet): Packet object that represents the packet before the current packet
"""
if (previousPacket.isExistingObject()
and self.packet.markerId == previousPacket.markerId):
isPostitionEqual = self.packet.isPostitionEqual(previousPacket)
if (not self.databaseWriteAccess
and isPostitionEqual
and self.packet.isMoving == 1
and previousPacket.positionTimestamp == previousPacket.timestamp):
# This is probably the exact same packet
self.packet.markerPrevPacketTimestamp = previousPacket.markerPrevPacketTimestamp
self.packet.markerCounter = previousPacket.markerCounter
self.packet.positionTimestamp = self.packet.timestamp
else:
self.packet.markerPrevPacketTimestamp = previousPacket.timestamp
self.packet.markerCounter = previousPacket.markerCounter + 1
if (isPostitionEqual):
self.packet.positionTimestamp = previousPacket.positionTimestamp
else:
self.packet.positionTimestamp = self.packet.timestamp
def _parseAssumedMoveTypeId(self, previousPacket):
"""Set current packet move type id
Args:
previousPacket (Packet): Packet object that represents the packet before the current packet
"""
packetAssumedMoveTypePolicy = PacketAssumedMoveTypePolicy(self.db)
self.packet.isMoving = packetAssumedMoveTypePolicy.getAssumedMoveType(self.packet, previousPacket)
def _parsePacketTail(self, previousPacket):
"""Set current packet tail timestamp
Args:
previousPacket (Packet): Packet object that represents the packet before the current packet
"""
packetTailPolicy = PacketTailPolicy(self.packet, previousPacket)
self.packet.packetTailTimestamp = packetTailPolicy.getPacketTailTimestamp()
def _parsePacketRelatedMapSectors(self, previousPacket):
"""Set current packet related map sectors
Args:
previousPacket (Packet): Packet object that represents the packet before the current packet
"""
packetRelatedMapSectorsPolicy = PacketRelatedMapSectorsPolicy(
self.packetRepository)
self.packet.relatedMapSectors = packetRelatedMapSectorsPolicy.getAllRelatedMapSectors(
self.packet, previousPacket)
def _getNewMarkerId(self):
"""Creates a new marker id
Returns:
int
"""
if (not self.databaseWriteAccess):
return 1
else:
# No suitable marker id found, let's create a new!
marker = self.markerRepository.create()
marker.save()
return marker.id