trackdirect2/server/trackdirect/parser/AprsPacketParser.py

549 lines
27 KiB
Python

import logging
import datetime
import time
import calendar
import hashlib
from server.trackdirect.exceptions.TrackDirectParseError import TrackDirectParseError
from server.trackdirect.exceptions.TrackDirectMissingSenderError import TrackDirectMissingSenderError
from server.trackdirect.exceptions.TrackDirectMissingStationError import TrackDirectMissingStationError
from server.trackdirect.repositories.OgnHiddenStationRepository import OgnHiddenStationRepository
from server.trackdirect.repositories.OgnDeviceRepository import OgnDeviceRepository
from server.trackdirect.repositories.StationRepository import StationRepository
from server.trackdirect.repositories.SenderRepository import SenderRepository
from server.trackdirect.repositories.PacketRepository import PacketRepository
from server.trackdirect.repositories.PacketTelemetryRepository import PacketTelemetryRepository
from server.trackdirect.repositories.PacketWeatherRepository import PacketWeatherRepository
from server.trackdirect.repositories.PacketOgnRepository import PacketOgnRepository
from server.trackdirect.repositories.MarkerRepository import MarkerRepository
from server.trackdirect.repositories.StationTelemetryBitsRepository import StationTelemetryBitsRepository
from server.trackdirect.repositories.StationTelemetryEqnsRepository import StationTelemetryEqnsRepository
from server.trackdirect.repositories.StationTelemetryParamRepository import StationTelemetryParamRepository
from server.trackdirect.repositories.StationTelemetryUnitRepository import StationTelemetryUnitRepository
from server.trackdirect.objects.Packet import Packet
from server.trackdirect.parser.policies.AprsPacketTypePolicy import AprsPacketTypePolicy
from server.trackdirect.parser.policies.PacketAssumedMoveTypePolicy import PacketAssumedMoveTypePolicy
from server.trackdirect.parser.policies.PreviousPacketPolicy import PreviousPacketPolicy
from server.trackdirect.parser.policies.PacketTailPolicy import PacketTailPolicy
from server.trackdirect.parser.policies.PacketRelatedMapSectorsPolicy import PacketRelatedMapSectorsPolicy
from server.trackdirect.parser.policies.PacketMapIdPolicy import PacketMapIdPolicy
from server.trackdirect.parser.policies.PacketPathPolicy import PacketPathPolicy
from server.trackdirect.parser.policies.MapSectorPolicy import MapSectorPolicy
from server.trackdirect.parser.policies.PacketCommentPolicy import PacketCommentPolicy
from server.trackdirect.parser.policies.PacketKillCharPolicy import PacketKillCharPolicy
from server.trackdirect.parser.policies.StationNameFormatPolicy import StationNameFormatPolicy
from server.trackdirect.parser.policies.PacketOgnDataPolicy import PacketOgnDataPolicy
class AprsPacketParser:
"""AprsPacketParser takes an aprslib output and converts it to a Track direct Packet."""
def __init__(self, db, save_ogn_stations_with_missing_identity):
"""Initialize the AprsPacketParser class.
Args:
db (psycopg2.Connection): Database connection
save_ogn_stations_with_missing_identity (bool): True if we should not ignore stations with a missing identity
"""
self.is_hidden_station = None
self.data = None
self.packet = None
self.db = db
self.save_ogn_stations_with_missing_identity = save_ogn_stations_with_missing_identity
self.logger = logging.getLogger('trackdirect')
self.database_write_access = True
self.source_id = 1
self.ogn_device_repository = OgnDeviceRepository(db)
self.ogn_hidden_station_repository = OgnHiddenStationRepository(db)
self.station_repository = StationRepository(db)
self.sender_repository = SenderRepository(db)
self.packet_repository = PacketRepository(db)
self.packet_telemetry_repository = PacketTelemetryRepository(db)
self.packet_weather_repository = PacketWeatherRepository(db)
self.packet_ogn_repository = PacketOgnRepository(db)
self.markerRepository = MarkerRepository(db)
self.station_telemetry_bits_repository = StationTelemetryBitsRepository(db)
self.station_telemetry_eqns_repository = StationTelemetryEqnsRepository(db)
self.station_telemetry_param_repository = StationTelemetryParamRepository(db)
self.station_telemetry_unit_repository = StationTelemetryUnitRepository(db)
def set_database_write_access(self, database_write_access):
"""Enable or disable database updates.
Args:
database_write_access (bool): True if we have database access otherwise false
"""
self.database_write_access = database_write_access
def set_source_id(self, source_id):
"""Set what source packet is from (APRS, CWOP ...).
Args:
source_id (int): Id that corresponds to id in source-table
"""
self.source_id = source_id
def get_packet(self, data, timestamp=None, minimal=False):
"""Returns the resulting packet.
Args:
data (dict): Raw packet data
timestamp (int): Packet timestamp
minimal (bool): Set to true to only perform minimal parsing
Returns:
Packet
"""
self.is_hidden_station = False
self.data = data
self.packet = Packet(self.db)
self.packet.source_id = self.source_id
self._parse_packet_initial_values(timestamp)
self._parse_packet_reported_timestamp()
# Parse OGN stuff before station-id, may result in no station to create
self._parse_packet_ogn()
if self.packet.map_id != 15:
self._parse_packet_type()
self._parse_packet_sender()
self._parse_packet_station_name()
self._parse_packet_station_id()
self._parse_packet_rng()
self._parse_packet_phg()
if self.packet.station_id is None:
self.packet.map_id = 4
else:
self._parse_packet_comment()
self._parse_packet_weather()
self._parse_packet_telemetry_definition()
self._parse_packet_telemetry()
self._parse_packet_position()
self._parse_packet_path()
if not minimal:
previous_packet_policy = PreviousPacketPolicy(self.packet, self.db)
previous_packet = previous_packet_policy.get_previous_packet()
self._parse_assumed_move_type_id(previous_packet)
self._parse_packet_tail(previous_packet)
self._parse_packet_map_id(previous_packet)
self._parse_packet_previous_timestamps(previous_packet)
self._parse_packet_phg_rng_timestamps(previous_packet)
self._parse_packet_related_map_sectors(previous_packet)
if self.is_hidden_station:
if self.packet.ogn is not None:
self.packet.ogn.ogn_address_type_id = None
self.packet.ogn.ogn_sender_address = None
self.packet.comment = None
self.packet.raw = None
return self.packet
def _parse_packet_initial_values(self, timestamp):
"""Set packet initial values.
Args:
timestamp (int): Packet timestamp
"""
self.packet.raw = self.data['raw'].replace('\x00', '')
self.packet.symbol = self.data.get('symbol')
self.packet.symbol_table = self.data.get('symbol_table')
self.packet.reported_timestamp = self.data.get('timestamp')
self.packet.raw_path = f"{self.data['to']},{','.join(self.data['path'])}" if 'path' in self.data and 'to' in self.data else None
self.packet.timestamp = timestamp if timestamp is not None else int(time.time())
self.packet.position_timestamp = self.packet.timestamp
self.packet.packet_tail_timestamp = 0
self.packet.posambiguity = self.data.get('posambiguity')
self.packet.speed = self.data.get('speed')
self.packet.course = self.data.get('course')
self.packet.altitude = self.data.get('altitude')
self.packet.map_id = 10 # Default map for a packet without a position
self.packet.marker_id = 1
self.packet.marker_counter = 1 # We always start at 1
def _parse_packet_reported_timestamp(self):
"""Set packet reported timestamp."""
if 'timestamp' in self.data:
reported_timestamp_day = int(datetime.datetime.utcfromtimestamp(int(self.data['timestamp'])).strftime('%d'))
current_timestamp_day = int(datetime.datetime.utcfromtimestamp(self.packet.timestamp).strftime('%d'))
if reported_timestamp_day > current_timestamp_day and self.data['timestamp'] > self.packet.timestamp:
# Day is in the future, the reported timestamp was probably in previous month
prev_month = int(datetime.datetime.utcfromtimestamp(int(self.data['timestamp']) - (reported_timestamp_day + 1) * 24 * 60 * 60).strftime('%m'))
prev_month_year = int(datetime.datetime.utcfromtimestamp(int(self.data['timestamp']) - (reported_timestamp_day + 1) * 24 * 60 * 60).strftime('%Y'))
number_of_days_in_month = int(calendar.monthrange(prev_month_year, prev_month)[1])
self.packet.reported_timestamp = int(self.data['timestamp']) - (number_of_days_in_month * 24 * 60 * 60)
else:
self.packet.reported_timestamp = None
def _parse_packet_rng(self):
"""Set packet RNG data."""
if 'rng' in self.data:
try:
self.packet.rng = float(self.data['rng'])
self.packet.latest_rng_timestamp = self.packet.timestamp
if self.packet.rng <= 0:
self.packet.rng = None
except ValueError:
# Not a float
self.packet.rng = None
def _parse_packet_phg(self):
"""Set packet PHG data."""
if 'phg' in self.data:
phg = str(self.data['phg'])[0:4]
if len(phg) == 4 and phg.isdigit():
self.packet.phg = phg
self.packet.latest_phg_timestamp = self.packet.timestamp
def _parse_packet_sender(self):
"""Set sender id and name."""
station_name_format_policy = StationNameFormatPolicy()
self.packet.senderName = station_name_format_policy.get_correct_format(self.data["from"])
try:
sender = self.sender_repository.get_cached_object_by_name(self.packet.senderName)
self.packet.sender_id = sender.id
except TrackDirectMissingSenderError:
if self.database_write_access:
sender = self.sender_repository.get_object_by_name(self.packet.senderName, True, self.packet.source_id)
self.packet.sender_id = sender.id
def _parse_packet_station_name(self):
"""Set station name."""
if "object_name" in self.data and self.data["object_name"]:
self.packet.station_type_id = 2
station_name_format_policy = StationNameFormatPolicy()
self.packet.stationName = station_name_format_policy.get_correct_format(self.data["object_name"])
if not self.packet.stationName:
self.packet.stationName = self.packet.senderName
if self.packet.stationName == self.packet.senderName:
self.packet.station_type_id = 1
else:
self.packet.station_type_id = 1
self.packet.stationName = self.packet.senderName
def _parse_packet_station_id(self):
"""Set station id."""
if self.packet.stationName is None:
return
try:
station = self.station_repository.get_cached_object_by_name(self.packet.stationName, self.packet.source_id)
self.packet.station_id = station.id
if self.packet.stationName != station.name:
# Station lower/upper case is modified
station.name = self.packet.stationName
station.save()
if station.station_type_id != self.packet.station_type_id and int(self.packet.station_type_id) == 1:
# Switch to station type 1
station.station_type_id = self.packet.station_type_id
station.save()
except TrackDirectMissingStationError:
if self.database_write_access:
station = self.station_repository.get_object_by_name(self.packet.stationName, self.packet.source_id, self.packet.station_type_id, True)
self.packet.station_id = station.id
def _parse_packet_comment(self):
"""Set packet comment."""
comment_policy = PacketCommentPolicy()
self.packet.comment = comment_policy.get_comment(self.data, self.packet.packet_type_id)
def _parse_packet_ogn(self):
"""Set the OGN sender address."""
ogn_data_policy = PacketOgnDataPolicy(self.data, self.ogn_device_repository, self.packet.source_id)
if ogn_data_policy.is_ogn_position_packet:
original_raw = self.packet.raw
if not ogn_data_policy.is_allowed_to_track:
self.packet.map_id = 15
return
elif not ogn_data_policy.is_allowed_to_identify:
if not self.save_ogn_stations_with_missing_identity:
self.packet.map_id = 15
return
self.is_hidden_station = True
self.data["from"] = self._get_hidden_station_name()
self.data["object_name"] = None
self.data['ogn'] = ogn_data_policy.get_ogn_data()
if self.data['ogn'] is not None:
self.packet.ogn = self.packet_ogn_repository.get_object_from_packet_data(self.data)
self.packet.ogn.station_id = self.packet.station_id
self.packet.ogn.timestamp = self.packet.timestamp
self.data["comment"] = None
self._modify_ogn_symbol(original_raw)
def _modify_ogn_symbol(self, original_raw):
"""Sets a better symbol for an OGN station.
Args:
original_raw (str): Non modified raw
"""
if (self.packet.symbol == '\'' and self.packet.symbol_table == '/') or (self.packet.symbol == '^' and self.packet.symbol_table in ['/', '\\']) or (self.packet.symbol == 'g' and self.packet.symbol_table in ['/']):
ogn_device = None
if self.packet.ogn is not None and self.packet.ogn.ogn_sender_address is not None:
ogn_device = self.ogn_device_repository.get_object_by_device_id(self.packet.ogn.ogn_sender_address)
if ogn_device is not None and ogn_device.is_existing_object() and 0 < ogn_device.ddb_aircraft_type < 6:
# If another aircraft type exist in device db, use that instead
if ogn_device.ddb_aircraft_type == 1: # Gliders/motoGliders
# Glider -> Glider
self.packet.symbol = '^'
self.packet.symbol_table = 'G'
elif ogn_device.ddb_aircraft_type == 2: # Planes
# Powered aircraft -> Propeller aircraft
self.packet.symbol = '^'
self.packet.symbol_table = 'P'
elif ogn_device.ddb_aircraft_type == 3: # Ultralights
# Small plane
self.packet.symbol = '\''
self.packet.symbol_table = '/'
elif ogn_device.ddb_aircraft_type == 4: # Helicopters
# Helicopter
self.packet.symbol = 'X'
self.packet.symbol_table = '/'
elif ogn_device.ddb_aircraft_type == 5: # Drones/UAV
# UAV -> Drone
self.packet.symbol = '^'
self.packet.symbol_table = 'D'
elif self.packet.ogn is not None and self.packet.ogn.ogn_aircraft_type_id is not None:
if self.packet.ogn.ogn_aircraft_type_id == 1:
# Glider -> Glider
self.packet.symbol = '^'
self.packet.symbol_table = 'G'
elif self.packet.ogn.ogn_aircraft_type_id == 9:
# Jet aircraft -> Jet
self.packet.symbol = '^'
self.packet.symbol_table = 'J'
elif self.packet.ogn.ogn_aircraft_type_id == 8:
# Powered aircraft -> Propeller aircraft
self.packet.symbol = '^'
self.packet.symbol_table = 'P'
elif self.packet.ogn.ogn_aircraft_type_id == 13:
# UAV -> Drone
self.packet.symbol = '^'
self.packet.symbol_table = 'D'
elif self.packet.ogn.ogn_aircraft_type_id == 7: # Paraglider
# Map to own symbol 94-76.svg (do not show hangglider symbol 103-1.svg, 'g' = 103)
self.packet.symbol = '^' # 94
self.packet.symbol_table = 'L' # 76
if (self.packet.symbol == '\'' and self.packet.symbol_table == '/') or (self.packet.symbol == '^' and self.packet.symbol_table in ['/', '\\']):
# Current symbol is still "small aircraft" or "large aircraft"
if original_raw.startswith('FMT'):
# FMT, Remotely Piloted
self.packet.symbol = '^'
self.packet.symbol_table = 'R'
def _get_hidden_station_name(self):
"""Returns a unidentifiable station name.
Returns:
str
"""
date = datetime.datetime.utcfromtimestamp(self.packet.timestamp).strftime('%Y%m%d')
daily_station_name_hash = hashlib.sha256((self.data["from"] + date).encode()).hexdigest()
ogn_hidden_station = self.ogn_hidden_station_repository.get_object_by_hashed_name(daily_station_name_hash, True)
return ogn_hidden_station.get_station_name()
def _parse_packet_weather(self):
"""Parse weather data."""
if "weather" in self.data:
self.packet.weather = self.packet_weather_repository.get_object_from_packet_data(self.data)
self.packet.weather.station_id = self.packet.station_id
self.packet.weather.timestamp = self.packet.timestamp
def _parse_packet_telemetry(self):
"""Parse telemetry data."""
if "telemetry" in self.data:
self.packet.telemetry = self.packet_telemetry_repository.get_object_from_packet_data(self.data)
self.packet.telemetry.station_id = self.packet.station_id
self.packet.telemetry.timestamp = self.packet.timestamp
def _parse_packet_telemetry_definition(self):
"""Parse telemetry definition."""
if "tBITS" in self.data:
self.packet.station_telemetry_bits = self.station_telemetry_bits_repository.get_object_from_packet_data(self.data)
self.packet.station_telemetry_bits.station_id = self.packet.station_id
self.packet.station_telemetry_bits.timestamp = self.packet.timestamp
if "tEQNS" in self.data:
self.packet.station_telemetry_eqns = self.station_telemetry_eqns_repository.get_object_from_packet_data(self.data)
self.packet.station_telemetry_eqns.station_id = self.packet.station_id
self.packet.station_telemetry_eqns.timestamp = self.packet.timestamp
if "tPARM" in self.data:
self.packet.station_telemetry_param = self.station_telemetry_param_repository.get_object_from_packet_data(self.data)
self.packet.station_telemetry_param.station_id = self.packet.station_id
self.packet.station_telemetry_param.timestamp = self.packet.timestamp
if "tUNIT" in self.data:
self.packet.station_telemetry_unit = self.station_telemetry_unit_repository.get_object_from_packet_data(self.data)
self.packet.station_telemetry_unit.station_id = self.packet.station_id
self.packet.station_telemetry_unit.timestamp = self.packet.timestamp
def _parse_packet_type(self):
"""Set packet type."""
aprs_packet_type_policy = AprsPacketTypePolicy()
self.packet.packet_type_id = aprs_packet_type_policy.get_packet_type(self.packet)
def _parse_packet_position(self):
"""Parse the packet position and related attributes."""
if "latitude" in self.data and "longitude" in self.data and self.data["latitude"] is not None and self.data["longitude"] is not None:
self.packet.latitude = self.data['latitude']
self.packet.longitude = self.data['longitude']
map_sector_policy = MapSectorPolicy()
self.packet.map_sector = map_sector_policy.get_map_sector(self.data["latitude"], self.data["longitude"])
self.packet.map_id = 1 # Default map for a packet with a position
self.packet.is_moving = 1 # Moving/Stationary, default value is moving
def _parse_packet_path(self):
"""Parse packet path."""
if "path" in self.data and isinstance(self.data["path"], list):
packet_path_policy = PacketPathPolicy(self.data['path'], self.packet.source_id, self.station_repository, self.sender_repository)
self.packet.station_id_path = packet_path_policy.get_station_id_path()
self.packet.station_name_path = packet_path_policy.get_station_name_path()
self.packet.station_location_path = packet_path_policy.get_station_location_path()
def _get_packet_raw_body(self):
"""Returns packet raw body as string.
Returns:
str: Packet raw body as string
"""
try:
raw_header, raw_body = self.data["raw"].split(':', 1)
except ValueError:
raise TrackDirectParseError('Could not split packet into header and body', self.data)
if len(raw_body) == 0:
raise TrackDirectParseError('Packet body is empty', self.data)
return raw_body
def _parse_packet_phg_rng_timestamps(self, previous_packet):
"""Set current packet timestamp when latest rng / phg was received
Args:
previous_packet (Packet): Packet object that represents the packet before the current packet
"""
if previous_packet.is_existing_object() and self.packet.is_position_equal(previous_packet):
if (self.packet.phg is None and previous_packet.latest_phg_timestamp is not None
and previous_packet.latest_phg_timestamp > (self.packet.timestamp - 86400)):
self.latest_phg_timestamp = previous_packet.latest_phg_timestamp
if (self.packet.rng is None and previous_packet.latest_rng_timestamp is not None
and previous_packet.latest_rng_timestamp > (self.packet.timestamp - 86400)):
self.latest_rng_timestamp = previous_packet.latest_rng_timestamp
def _parse_packet_map_id(self, previous_packet):
"""Set current packet map id and related
Args:
previous_packet (Packet): Packet object that represents the packet before the current packet
"""
map_id_policy = PacketMapIdPolicy(self.packet, previous_packet)
kill_char_policy = PacketKillCharPolicy()
if kill_char_policy.has_kill_character(self.data):
map_id_policy.enable_having_kill_character()
if map_id_policy.is_replacing_previous_packet():
self.packet.replace_packet_id = previous_packet.id
self.packet.replace_packet_timestamp = previous_packet.timestamp
if map_id_policy.is_confirming_previous_packet():
self.packet.confirm_packet_id = previous_packet.id
self.packet.confirm_packet_timestamp = previous_packet.timestamp
if map_id_policy.is_killing_previous_packet():
self.packet.abnormal_packet_id = previous_packet.id
self.packet.abnormal_packet_timestamp = previous_packet.timestamp
self.packet.map_id = map_id_policy.get_map_id()
self.packet.marker_id = map_id_policy.get_marker_id()
if self.packet.marker_id is None:
# Policy could not find a marker id to use, create a new
self.packet.marker_id = self._get_new_marker_id()
if self.packet.marker_id == 1:
self.packet.map_id = 4
def _parse_packet_previous_timestamps(self, previous_packet):
"""Set packet previous timestamps and related
Args:
previous_packet (Packet): Packet object that represents the packet before the current packet
"""
if previous_packet.is_existing_object() and self.packet.marker_id == previous_packet.marker_id:
is_position_equal = self.packet.is_position_equal(previous_packet)
if (not self.database_write_access and is_position_equal and self.packet.is_moving == 1
and previous_packet.position_timestamp == previous_packet.timestamp):
# This is probably the exact same packet
self.packet.marker_prev_packet_timestamp = previous_packet.marker_prev_packet_timestamp
self.packet.marker_counter = previous_packet.marker_counter
self.packet.position_timestamp = self.packet.timestamp
else:
self.packet.marker_prev_packet_timestamp = previous_packet.timestamp
self.packet.marker_counter = previous_packet.marker_counter + 1
if is_position_equal:
self.packet.position_timestamp = previous_packet.position_timestamp
else:
self.packet.position_timestamp = self.packet.timestamp
def _parse_assumed_move_type_id(self, previous_packet):
"""Set current packet move type id
Args:
previous_packet (Packet): Packet object that represents the packet before the current packet
"""
packet_assumed_move_type_policy = PacketAssumedMoveTypePolicy(self.db)
self.packet.is_moving = packet_assumed_move_type_policy.get_assumed_move_type(self.packet, previous_packet)
def _parse_packet_tail(self, previous_packet):
"""Set current packet tail timestamp
Args:
previous_packet (Packet): Packet object that represents the packet before the current packet
"""
packet_tail_policy = PacketTailPolicy(self.packet, previous_packet)
self.packet.packet_tail_timestamp = packet_tail_policy.get_packet_tail_timestamp()
def _parse_packet_related_map_sectors(self, previous_packet):
"""Set current packet related map sectors
Args:
previous_packet (Packet): Packet object that represents the packet before the current packet
"""
packet_related_map_sectors_policy = PacketRelatedMapSectorsPolicy(self.packet_repository)
self.packet.related_map_sectors = packet_related_map_sectors_policy.get_all_related_map_sectors(self.packet, previous_packet)
def _get_new_marker_id(self):
"""Creates a new marker id
Returns:
int
"""
if not self.database_write_access:
return 1
else:
# No suitable marker id found, let's create a new!
marker = self.markerRepository.create()
marker.save()
return marker.id