From e01254df212a3f215169ed8036430fc6c05a40dc Mon Sep 17 00:00:00 2001 From: Per Qvarforth Date: Sun, 30 Jan 2022 16:47:48 +0100 Subject: [PATCH] improved configuration --- config/trackdirect.ini | 38 +++--- htdocs/includes/common.php | 31 +++++ htdocs/public/index.php | 116 ++++++++++-------- server/bin/collector.py | 7 +- server/trackdirect/TrackDirectConfig.py | 32 ++++- .../trackdirect/TrackDirectDataCollector.py | 11 +- server/trackdirect/parser/AprsPacketParser.py | 9 +- .../websocket/aprsis/AprsISPayloadCreator.py | 6 +- 8 files changed, 166 insertions(+), 84 deletions(-) diff --git a/config/trackdirect.ini b/config/trackdirect.ini index ba50fed..769dd48 100644 --- a/config/trackdirect.ini +++ b/config/trackdirect.ini @@ -12,11 +12,15 @@ password="foobar" port="5432" ;; Settings for the remover script -days_to_save_position_data="31" -days_to_save_station_data="90" +days_to_save_position_data="10" +days_to_save_station_data="30" days_to_save_weather_data="10" days_to_save_telemetry_data="10" +;; If this setting is enabled, OGN stations that we are not allowed to reveal the identity of will be given a random name similar to "UNKNOWN123" +;; If disabled we will drop all packets regarding stations that we should not reveal the identity of. +save_ogn_stations_with_missing_identity="0" + [websocket_server] @@ -29,10 +33,10 @@ port="9000" ;; Websocket server log output error_log="~/trackdirect/server/log/wsserver_aprs.log" -;; Frequency limit -;; Packets received more frequently than the configured frequency limit will be dropped -;; Frequency limit is specified in seconds, and 0 means that the limit is disabled. -;; This setting is very useful when frequency is very high (when receiving data from the OGN network this needs to be at least 15s) +;; Packets received more frequently than the configured frequency limit will be dropped (limit is specified in seconds) +;; This frequency limit is only refering to pakets that is received in real time from the filtered feed used by the websocket server +;; This frequency limit may be a bit more forgiving than the frequence limit on the collector. +;; When receiving data from the OGN network this needs to be about 15s or more. frequency_limit="0" ;; First APRS IS server for the websocket server to connect to. @@ -40,6 +44,7 @@ frequency_limit="0" aprs_host1="127.0.0.1" aprs_port1="14580" +;; Important that you set the correct source, otherwise it might be handled incorrect ;; - Source Id 1: APRS-IS ;; - Source Id 2: CWOP ;; - Source Id 3: CBAPRS @@ -54,18 +59,18 @@ aprs_source_id1="1" ;aprs_source_id2="2" ;; Allow time travel -;; Use this settings to disable data requests with a time interval -;; Useful when it is not allowed to show data older than 24h (like when data comes from the OGN network) -;; Note that you need to configure the remover script to delete data after 24h as well (if the source require you to do so) +;; Use this settings to disable/enable data requests with a time interval (this must be disabled for the OGN network) allow_time_travel="1" -;; Max default time in seconds (how old packets that will be included in the response) +;; Max default time in minutes (how old packets that will be included in the response) +;; This setting should be no more than 1440 for for the OGN network. max_default_time="1440" -;; Max time in seconds when filtering (how old packets that will be included in the response) -max_filter_time="14400" +;; Max time in minutes when filtering (how old packets that will be included in the response) +;; This setting should be no more than 1440 for for the OGN network. +max_filter_time="1440" -;; Time in seconds until idle client is disconnected +;; Time in minutes until idle client is disconnected max_client_idle_time="60" ;; Max age in seconds for real time packets waiting to be sent to client (dropping packets if limit is excceded) @@ -78,6 +83,7 @@ host="127.0.0.1" port_full="10152" port_filtered="14580" +;; Important that you set the correct source, otherwise it might be handled incorrect ;; - Source Id 1: APRS-IS ;; - Source Id 2: CWOP ;; - Source Id 3: CBAPRS @@ -92,10 +98,8 @@ passcode="-1" ;; Database inserts is done in batches numbers_in_batch="50" -;; Frequency limit -;; Packets received more frequently than the configured frequency limit will not be shown on map. -;; Frequency limit is specified per station in seconds, and 0 means that the limit is disabled. -;; This setting is very useful when frequency is very high (when receiving data from the OGN network this needs to be at least 20s). +;; Packets received more frequently than the configured frequency limit will not be shown on map (limit is specified in seconds) +;; When receiving data from the OGN network this needs to be 20s or more. ;; If setting save_fast_packets to "0", packets that is received to frequently will not be save (useful for OGN, but not for APRS-IS). frequency_limit="5" save_fast_packets="1" diff --git a/htdocs/includes/common.php b/htdocs/includes/common.php index b8627bb..8f04af3 100644 --- a/htdocs/includes/common.php +++ b/htdocs/includes/common.php @@ -1744,4 +1744,35 @@ function getSymbolDescription($symbolTable, $symbol, $includeUndefinedOverlay) } } } +} + + +/** + * Returnes true if the time travel feature works + * + * @return boolean + */ +function isTimeTravelAllowed() { + $isTimeTravelAllowed = false; + $config = parse_ini_file(ROOT . '/../config/trackdirect.ini', true); + + if (isset($config['websocket_server'])) { + if (isset($config['websocket_server']['allow_time_travel'])) { + if ($config['websocket_server']['allow_time_travel'] == '1') { + $isTimeTravelAllowed = true; + } + } + + if (isset($config['websocket_server']['aprs_source_id1']) && $config['websocket_server']['aprs_source_id1'] == 5) { + // Data source is OGN, disable time travel (server will block it anyway) + $isTimeTravelAllowed = false; + } + + if (isset($config['websocket_server']['aprs_source_id2']) && $config['websocket_server']['aprs_source_id2'] == 5) { + // Data source is OGN, disable time travel (server will block it anyway) + $isTimeTravelAllowed = false; + } + } + + return $isTimeTravelAllowed; } \ No newline at end of file diff --git a/htdocs/public/index.php b/htdocs/public/index.php index 50bf9dc..414ace9 100755 --- a/htdocs/public/index.php +++ b/htdocs/public/index.php @@ -299,62 +299,70 @@ Travel in time
-

Select date and time to show map data for (enter time for your locale time zone). The regular time length select box can still be used to select how old data that should be shown (relative to selected date and time).

-

*Note that the heatmap will still based on data from the latest hour (not the selected date and time).

-

Date and time:

+ +
+

+ The time travel feature that allows you to see the map as it looked like an earlier date is disabled on this website. The reason is probably that it is a requirement from the data source used. +

+
+ +

Select date and time to show map data for (enter time for your locale time zone). The regular time length select box can still be used to select how old data that should be shown (relative to selected date and time).

+

*Note that the heatmap will still based on data from the latest hour (not the selected date and time).

+

Date and time:

-
- + + - - -
+ + + +
diff --git a/server/bin/collector.py b/server/bin/collector.py index 79ea1b8..d206d00 100644 --- a/server/bin/collector.py +++ b/server/bin/collector.py @@ -26,6 +26,10 @@ if __name__ == '__main__': collectorNumber = int(sys.argv[2]) collectorOptions = config.collector[collectorNumber] + saveOgnStationsWithMissingIdentity = False + if (config.saveOgnStationsWithMissingIdentity) : + saveOgnStationsWithMissingIdentity = True + fh = logging.handlers.RotatingFileHandler(filename=os.path.expanduser( collectorOptions['error_log']), mode='a', maxBytes=1000000, backupCount=10) fh.setLevel(logging.WARNING) @@ -47,7 +51,8 @@ if __name__ == '__main__': try: trackDirectDataCollector = trackdirect.TrackDirectDataCollector( - collectorOptions) + collectorOptions, + saveOgnStationsWithMissingIdentity) trackDirectDataCollector.run() except Exception as e: trackDirectLogger.error(e, exc_info=1) diff --git a/server/trackdirect/TrackDirectConfig.py b/server/trackdirect/TrackDirectConfig.py index 20cea9d..b8c2217 100644 --- a/server/trackdirect/TrackDirectConfig.py +++ b/server/trackdirect/TrackDirectConfig.py @@ -48,6 +48,15 @@ class TrackDirectConfig(Singleton): self.daysToSaveTelemetryData = int(configParser.get( 'database', 'days_to_save_telemetry_data').strip('"')) + self.saveOgnStationsWithMissingIdentity = False + try: + saveOgnStationsWithMissingIdentity = configParser.get( + 'database', 'save_ogn_stations_with_missing_identity').strip('"') + if (saveOgnStationsWithMissingIdentity == "1"): + self.saveOgnStationsWithMissingIdentity = True + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + pass + # Websocket server self.websocketHostname = configParser.get( 'websocket_server', 'host').strip('"') @@ -69,11 +78,11 @@ class TrackDirectConfig(Singleton): allowTimeTravel = configParser.get( 'websocket_server', 'allow_time_travel').strip('"') - self.allowTimeTravel = True - if (allowTimeTravel == "0"): - self.allowTimeTravel = False + self.allowTimeTravel = False + if (allowTimeTravel == "1"): + self.allowTimeTravel = True - # Websocket server APRS-IS connection + # Websocket server APRS connection (we support 2 different sources, more can be added...) try: self.websocketAprsHost1 = configParser.get( 'websocket_server', 'aprs_host1').strip('"') @@ -98,6 +107,14 @@ class TrackDirectConfig(Singleton): self.websocketAprsHost2 = None self.websocketAprsPort2 = None + if (self.websocketAprsSourceId1 == 5 or self.websocketAprsSourceId2 == 5) : + # At least one source is of type OGN, disable display of older data + self.allowTimeTravel = False + if (self.maxDefaultTime > 1440) : + self.maxDefaultTime = 1440 + if (self.maxFilterTime > 1440) : + self.maxDefaultTime = 1440 + # Collectors for collectorNumber in range(0, 5): self.collector[collectorNumber] = {} @@ -129,6 +146,13 @@ class TrackDirectConfig(Singleton): self.collector[collectorNumber]['error_log'] = configParser.get( 'collector' + str(collectorNumber), 'error_log').strip('"') + if (self.websocketAprsSourceId1 == 5 or self.websocketAprsSourceId2 == 5) : + # source is of type OGN, make sure we do not save to many packets (will cause to high load on db) + if (self.collector[collectorNumber]['frequency_limit'] < 10) : + self.collector[collectorNumber]['frequency_limit'] = 10 + self.collector[collectorNumber]['save_fast_packets'] = False + + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): self.collector[collectorNumber]['source_id'] = None self.collector[collectorNumber]['host'] = None diff --git a/server/trackdirect/TrackDirectDataCollector.py b/server/trackdirect/TrackDirectDataCollector.py index a5fa1c4..56188c9 100644 --- a/server/trackdirect/TrackDirectDataCollector.py +++ b/server/trackdirect/TrackDirectDataCollector.py @@ -19,7 +19,6 @@ from trackdirect.database.DatabaseConnection import DatabaseConnection from trackdirect.repositories.StationRepository import StationRepository from trackdirect.objects.Packet import Packet - class TrackDirectDataCollector(): """An TrackDirectDataCollector instance connects to the data source and saves all received packets to the database @@ -28,12 +27,14 @@ class TrackDirectDataCollector(): This is useful if you want one connection to the regular APRS-IS network and one connection to the CWOP network. """ - def __init__(self, collectorOptions): + def __init__(self, collectorOptions, saveOgnStationsWithMissingIdentity): """The __init__ method. Args: - collectorOptions (dict): Contains data like host, port, callsign, passcode, source id + collectorOptions (dict): Contains data like host, port, callsign, passcode, source id + saveOgnStationsWithMissingIdentity (boolean): True if we should not ignore stationss with a missing identity """ + self.saveOgnStationsWithMissingIdentity = saveOgnStationsWithMissingIdentity self.sourceHostname = collectorOptions['host'] self.sourcePort = collectorOptions['port_full'] self.numbersInBatch = collectorOptions['numbers_in_batch'] @@ -142,7 +143,7 @@ class TrackDirectDataCollector(): 'Collector has a delay on %s seconds', self.delay) packetDict = aprslib.parse(line) - parser = AprsPacketParser(self.db) + parser = AprsPacketParser(self.db, self.saveOgnStationsWithMissingIdentity) parser.setSourceId(self.sourceId) packet = parser.getPacket(packetDict, timestamp) @@ -177,7 +178,7 @@ class TrackDirectDataCollector(): try: line = line.decode('utf-8', 'ignore') packetDict = self.basicParse(line) - parser = AprsPacketParser(self.db) + parser = AprsPacketParser(self.db, self.saveOgnStationsWithMissingIdentity) parser.setSourceId(self.sourceId) packet = parser.getPacket(packetDict, timestamp, True) packet.markerId = 1 diff --git a/server/trackdirect/parser/AprsPacketParser.py b/server/trackdirect/parser/AprsPacketParser.py index 70a572c..80aee97 100644 --- a/server/trackdirect/parser/AprsPacketParser.py +++ b/server/trackdirect/parser/AprsPacketParser.py @@ -51,13 +51,15 @@ class AprsPacketParser(): """AprsPacketParser tackes a aprslib output and converts it to a Trackdirect Packet """ - def __init__(self, db): + def __init__(self, db, saveOgnStationsWithMissingIdentity): """The __init__ method. Args: - db (psycopg2.Connection): Database connection + db (psycopg2.Connection): Database connection + saveOgnStationsWithMissingIdentity (boolean): True if we should not ignore stationss with a missing identity """ self.db = db + self.saveOgnStationsWithMissingIdentity = saveOgnStationsWithMissingIdentity self.logger = logging.getLogger('trackdirect') self.databaseWriteAccess = True @@ -315,6 +317,9 @@ class AprsPacketParser(): return elif (not ognDataPolicy.isAllowedToIdentify): + if (not self.saveOgnStationsWithMissingIdentity) : + self.packet.mapId = 15 + return self.isHiddenStation = True self.data["from"] = self._getHiddenStationName() self.data["object_name"] = None diff --git a/server/trackdirect/websocket/aprsis/AprsISPayloadCreator.py b/server/trackdirect/websocket/aprsis/AprsISPayloadCreator.py index 0c7b44f..e38c573 100644 --- a/server/trackdirect/websocket/aprsis/AprsISPayloadCreator.py +++ b/server/trackdirect/websocket/aprsis/AprsISPayloadCreator.py @@ -42,6 +42,10 @@ class AprsISPayloadCreator(): self.config = TrackDirectConfig() self.stationHashTimestamps = {} + self.saveOgnStationsWithMissingIdentity = False + if (self.config.saveOgnStationsWithMissingIdentity) : + self.saveOgnStationsWithMissingIdentity = True + def getPayloads(self, line, sourceId): """Takes a raw packet and returnes a dict with the parsed result @@ -85,7 +89,7 @@ class AprsISPayloadCreator(): Packet """ basicPacketDict = aprslib.parse(line) - parser = AprsPacketParser(self.db) + parser = AprsPacketParser(self.db, self.saveOgnStationsWithMissingIdentity) parser.setDatabaseWriteAccess(False) parser.setSourceId(sourceId) try :