From b1f17027d52f854bdadb6f1b666723f210bc72f2 Mon Sep 17 00:00:00 2001 From: Peter Buchegger Date: Wed, 2 Feb 2022 15:27:22 +0100 Subject: [PATCH 1/7] add logging to stdout --- server/bin/collector.py | 16 ++++++++++------ server/bin/heatmapcreator.py | 5 ++++- server/bin/remover.py | 6 ++++-- server/bin/stationremover.py | 5 ++++- server/bin/wsserver.py | 9 +++++++-- 5 files changed, 29 insertions(+), 12 deletions(-) diff --git a/server/bin/collector.py b/server/bin/collector.py index d206d00..cc1706c 100644 --- a/server/bin/collector.py +++ b/server/bin/collector.py @@ -20,31 +20,35 @@ if __name__ == '__main__': config = trackdirect.TrackDirectConfig() config.populate(sys.argv[1]) - if (len(sys.argv) < 3) : + if (len(sys.argv) < 3): collectorNumber = 0 - else : + else: collectorNumber = int(sys.argv[2]) collectorOptions = config.collector[collectorNumber] saveOgnStationsWithMissingIdentity = False - if (config.saveOgnStationsWithMissingIdentity) : + if (config.saveOgnStationsWithMissingIdentity): saveOgnStationsWithMissingIdentity = True fh = logging.handlers.RotatingFileHandler(filename=os.path.expanduser( collectorOptions['error_log']), mode='a', maxBytes=1000000, backupCount=10) - fh.setLevel(logging.WARNING) formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') fh.setFormatter(formatter) + consoleHandler = logging.StreamHandler() + consoleHandler.setFormatter(formatter) + trackDirectLogger = logging.getLogger('trackdirect') trackDirectLogger.addHandler(fh) - trackDirectLogger.setLevel(logging.WARNING) + trackDirectLogger.addHandler(consoleHandler) + trackDirectLogger.setLevel(logging.INFO) aprslibLogger = logging.getLogger('aprslib.IS') aprslibLogger.addHandler(fh) - aprslibLogger.setLevel(logging.WARNING) + aprslibLogger.addHandler(consoleHandler) + aprslibLogger.setLevel(logging.INFO) trackDirectLogger.warning("Starting (Collecting from " + collectorOptions['host'] + ":" + str( collectorOptions['port_full']) + " using " + collectorOptions['callsign'] + " and " + str(collectorOptions['passcode']) + ")") diff --git a/server/bin/heatmapcreator.py b/server/bin/heatmapcreator.py index 0835f7d..683c8f3 100644 --- a/server/bin/heatmapcreator.py +++ b/server/bin/heatmapcreator.py @@ -22,14 +22,17 @@ if __name__ == '__main__': fh = logging.handlers.RotatingFileHandler(filename=os.path.expanduser( '~/trackdirect/server/log/heatmap.log'), mode='a', maxBytes=1000000, backupCount=10) - fh.setLevel(logging.INFO) formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') fh.setFormatter(formatter) + consoleHandler = logging.StreamHandler() + consoleHandler.setFormatter(formatter) + trackDirectLogger = logging.getLogger('trackdirect') trackDirectLogger.addHandler(fh) + trackDirectLogger.addHandler(consoleHandler) trackDirectLogger.setLevel(logging.INFO) trackDirectLogger.info("Starting (output directory: " + sys.argv[2] + ")") diff --git a/server/bin/remover.py b/server/bin/remover.py index d3a297b..1116319 100644 --- a/server/bin/remover.py +++ b/server/bin/remover.py @@ -37,14 +37,17 @@ if __name__ == '__main__': try: fh = logging.handlers.RotatingFileHandler(filename=os.path.expanduser( '~/trackdirect/server/log/remover_' + config.dbName + '.log'), mode='a', maxBytes=1000000, backupCount=10) - fh.setLevel(logging.INFO) formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') fh.setFormatter(formatter) + consoleHandler = logging.StreamHandler() + consoleHandler.setFormatter(formatter) + trackDirectLogger = logging.getLogger('trackdirect') trackDirectLogger.addHandler(fh) + trackDirectLogger.addHandler(consoleHandler) trackDirectLogger.setLevel(logging.INFO) trackDirectLogger.info("Starting") @@ -230,7 +233,6 @@ if __name__ == '__main__': if (deletedRows > 0): trackDirectLogger.info("Deleted %s stations" % (deletedRows)) - cursor.execute("""VACUUM ANALYZE station""") cursor.execute("""REINDEX TABLE station""") cursor.execute("""VACUUM ANALYZE sender""") diff --git a/server/bin/stationremover.py b/server/bin/stationremover.py index fca5bb4..1a7a110 100644 --- a/server/bin/stationremover.py +++ b/server/bin/stationremover.py @@ -33,14 +33,17 @@ if __name__ == '__main__': try: fh = logging.handlers.RotatingFileHandler(filename=os.path.expanduser( '~/trackdirect/server/log/stationremover.log'), mode='a', maxBytes=1000000, backupCount=10) - fh.setLevel(logging.INFO) formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') fh.setFormatter(formatter) + consoleHandler = logging.StreamHandler() + consoleHandler.setFormatter(formatter) + trackDirectLogger = logging.getLogger('trackdirect') trackDirectLogger.addHandler(fh) + trackDirectLogger.addHandler(consoleHandler) trackDirectLogger.setLevel(logging.INFO) trackDirectLogger.info("Starting") diff --git a/server/bin/wsserver.py b/server/bin/wsserver.py index d60674e..7f906c8 100644 --- a/server/bin/wsserver.py +++ b/server/bin/wsserver.py @@ -116,20 +116,25 @@ if __name__ == '__main__': fh = logging.handlers.RotatingFileHandler(filename=os.path.expanduser( config.errorLog), mode='a', maxBytes=1000000, backupCount=10) - fh.setLevel(logging.WARNING) fh.setFormatter(formatter) + consoleHandler = logging.StreamHandler() + consoleHandler.setFormatter(formatter) + trackDirectLogger = logging.getLogger('trackdirect') trackDirectLogger.addHandler(fh) + trackDirectLogger.addHandler(consoleHandler) + trackDirectLogger.setLevel(logging.INFO) fh2 = logging.handlers.RotatingFileHandler(filename=os.path.expanduser( config.errorLog), mode='a', maxBytes=1000000, backupCount=10) # aprslib is logging non important "socket error on ..." using ERROR-level - fh2.setLevel(logging.CRITICAL) fh2.setFormatter(formatter) aprslibLogger = logging.getLogger('aprslib.IS') aprslibLogger.addHandler(fh2) + aprslibLogger.addHandler(consoleHandler) + aprslibLogger.setLevel(logging.INFO) if options.fd is not None: worker(options, trackDirectLogger) From 4897dfe091784a1d471841e7ba2b1bfdb64402db Mon Sep 17 00:00:00 2001 From: Peter Buchegger Date: Wed, 2 Feb 2022 16:34:22 +0100 Subject: [PATCH 2/7] run socketserver forever --- server/bin/wsserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/bin/wsserver.py b/server/bin/wsserver.py index 7f906c8..702c3e9 100644 --- a/server/bin/wsserver.py +++ b/server/bin/wsserver.py @@ -47,7 +47,7 @@ def master(options, trackDirectLogger): childFDs={0: 0, 1: 1, 2: 2, port.fileno(): port.fileno()}, env=os.environ) - # reactor.run() + reactor.run() except Exception as e: trackDirectLogger.error(e, exc_info=1) From c4858e712fb46d10f2e32604bdbad3bd2948d3c9 Mon Sep 17 00:00:00 2001 From: Peter Buchegger Date: Wed, 2 Feb 2022 16:34:44 +0100 Subject: [PATCH 3/7] add newline --- server/trackdirect/TrackDirectWebsocketServer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/trackdirect/TrackDirectWebsocketServer.py b/server/trackdirect/TrackDirectWebsocketServer.py index eccbe97..133a6e9 100644 --- a/server/trackdirect/TrackDirectWebsocketServer.py +++ b/server/trackdirect/TrackDirectWebsocketServer.py @@ -435,4 +435,4 @@ class TrackDirectWebsocketServer(WebSocketServerProtocol): self.connectionState.setLatestMinutes(request["minutes"], None) if ("noRealTime" in request): - self.connectionState.disableRealTime() \ No newline at end of file + self.connectionState.disableRealTime() From b9d21774686a1089bb7cf770feb49399a7bba06d Mon Sep 17 00:00:00 2001 From: Peter Buchegger Date: Wed, 2 Feb 2022 16:35:03 +0100 Subject: [PATCH 4/7] run processes forever --- server/scripts/collector.sh | 2 +- server/scripts/heatmapcreator.sh | 2 +- server/scripts/wsserver.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/scripts/collector.sh b/server/scripts/collector.sh index c9d4bba..1abf8b9 100755 --- a/server/scripts/collector.sh +++ b/server/scripts/collector.sh @@ -17,6 +17,6 @@ else export PYTHONPATH=$PYTHONPATH:$CURRENTDIR/../trackdirect:$CURRENTDIR/../../heatmap-2.2.1/ cd $CURRENTDIR/.. - python2 ./bin/collector.py $CONFIGFILE $COLLECTORNUMBER 2>&1 & + python2 ./bin/collector.py $CONFIGFILE $COLLECTORNUMBER exit 0 fi diff --git a/server/scripts/heatmapcreator.sh b/server/scripts/heatmapcreator.sh index f084b19..e53e055 100755 --- a/server/scripts/heatmapcreator.sh +++ b/server/scripts/heatmapcreator.sh @@ -17,6 +17,6 @@ else export PYTHONPATH=$PYTHONPATH:$CURRENTDIR/../trackdirect:$CURRENTDIR/../../heatmap-2.2.1/ cd $CURRENTDIR/.. - python2 $CURRENTDIR/../bin/heatmapcreator.py $CONFIGFILE $DESTDIR 2>&1 & + python2 $CURRENTDIR/../bin/heatmapcreator.py $CONFIGFILE $DESTDIR exit 0 fi diff --git a/server/scripts/wsserver.sh b/server/scripts/wsserver.sh index 322981b..974fe82 100755 --- a/server/scripts/wsserver.sh +++ b/server/scripts/wsserver.sh @@ -15,6 +15,6 @@ else export PYTHONPATH=$PYTHONPATH:$CURRENTDIR/../trackdirect:$CURRENTDIR/../../heatmap-2.2.1/ cd $CURRENTDIR/.. - python2 $CURRENTDIR/../bin/wsserver.py --config $CONFIGFILE 2>&1 & + python2 $CURRENTDIR/../bin/wsserver.py --config $CONFIGFILE exit 0 fi From 37d0cd162845247fe075b9f2ca5487f393d65dc5 Mon Sep 17 00:00:00 2001 From: Peter Buchegger Date: Wed, 2 Feb 2022 21:40:49 +0100 Subject: [PATCH 5/7] master thread should also do some work --- server/bin/wsserver.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/bin/wsserver.py b/server/bin/wsserver.py index 702c3e9..193b608 100644 --- a/server/bin/wsserver.py +++ b/server/bin/wsserver.py @@ -35,9 +35,8 @@ def master(options, trackDirectLogger): site = Site(root) port = reactor.listenTCP(config.websocketPort, site) - port.stopReading() - for i in range(options.workers): + for i in range(options.workers - 1): args = [sys.executable, "-u", __file__] args.extend(sys.argv[1:]) args.extend(["--fd", str(port.fileno()), "--cpuid", str(i)]) From 7164f0edadd13c81f9cdc88efaa37360a30a6708 Mon Sep 17 00:00:00 2001 From: Per Qvarforth Date: Wed, 2 Feb 2022 22:16:42 +0100 Subject: [PATCH 6/7] support for websocket external port --- config/trackdirect.ini | 9 ++++++--- server/bin/wsserver.py | 8 +++++--- server/trackdirect/TrackDirectConfig.py | 8 ++++++++ 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/config/trackdirect.ini b/config/trackdirect.ini index 769dd48..5fc8d1d 100644 --- a/config/trackdirect.ini +++ b/config/trackdirect.ini @@ -25,11 +25,14 @@ save_ogn_stations_with_missing_identity="0" [websocket_server] ;; Websocket server host -host="localhost" +host="0.0.0.0" -;; Websocket server port (must be same as port specified in index.php) +;; Websocket server port, must be same as port specified in index.php (if not using a reverse proxy) port="9000" +;; If you are using a reverse proxy you need to play with the "external_port" setting, otherwise it should be the same as "port" +external_port="9000" + ;; Websocket server log output error_log="~/trackdirect/server/log/wsserver_aprs.log" @@ -60,7 +63,7 @@ aprs_source_id1="1" ;; Allow time travel ;; Use this settings to disable/enable data requests with a time interval (this must be disabled for the OGN network) -allow_time_travel="1" +allow_time_travel="0" ;; Max default time in minutes (how old packets that will be included in the response) ;; This setting should be no more than 1440 for for the OGN network. diff --git a/server/bin/wsserver.py b/server/bin/wsserver.py index 193b608..57cdfab 100644 --- a/server/bin/wsserver.py +++ b/server/bin/wsserver.py @@ -26,7 +26,8 @@ def master(options, trackDirectLogger): try: factory = WebSocketServerFactory( - "ws://" + config.websocketHostname + ":" + str(config.websocketPort)) + "ws://" + config.websocketHostname + ":" + str(config.websocketPort), + externalPort = config.websocketExternalPort) factory.protocol = trackdirect.TrackDirectWebsocketServer resource = WebSocketResource(factory) @@ -66,7 +67,8 @@ def worker(options, trackDirectLogger): trackDirectLogger.warning("Starting worker with PID " + str(workerPid)) factory = WebSocketServerFactory( - "ws://" + config.websocketHostname + ":" + str(config.websocketPort)) + "ws://" + config.websocketHostname + ":" + str(config.websocketPort), + externalPort = config.websocketExternalPort) factory.protocol = trackdirect.TrackDirectWebsocketServer # Enable WebSocket extension "permessage-deflate". @@ -98,7 +100,7 @@ if __name__ == '__main__': parser = argparse.ArgumentParser( description='Track Direct WebSocket Server') parser.add_argument('--config', dest='config', type=str, default=None, - help='The Track Direct config file, e.g. aprsdirect.ini') + help='The Track Direct config file, e.g. trackdirect.ini') parser.add_argument('--workers', dest='workers', type=int, default=DEFAULT_WORKERS, help='Number of workers to spawn - should fit the number of (physical) CPU cores.') parser.add_argument('--fd', dest='fd', type=int, default=None, diff --git a/server/trackdirect/TrackDirectConfig.py b/server/trackdirect/TrackDirectConfig.py index b8c2217..035c7ee 100644 --- a/server/trackdirect/TrackDirectConfig.py +++ b/server/trackdirect/TrackDirectConfig.py @@ -62,6 +62,14 @@ class TrackDirectConfig(Singleton): 'websocket_server', 'host').strip('"') self.websocketPort = int(configParser.get( 'websocket_server', 'port').strip('"')) + + self.websocketExternalPort = self.websocketPort + try : + self.websocketExternalPort = int(configParser.get( + 'websocket_server', 'external_port').strip('"')) + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + pass + self.errorLog = configParser.get( 'websocket_server', 'error_log').strip('"') self.websocketFrequencyLimit = configParser.get( From d4716b606d7c2df493683d3916440fca81fc6512 Mon Sep 17 00:00:00 2001 From: Per Qvarforth Date: Wed, 2 Feb 2022 22:19:25 +0100 Subject: [PATCH 7/7] startup script fixes --- README.md | 10 +++++----- server/scripts/remover.sh | 2 +- server/scripts/stationremover.sh | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index fb1d9d9..0247f42 100644 --- a/README.md +++ b/README.md @@ -242,11 +242,11 @@ Note that the collector and wsserver shell scripts can be scheduled to start onc Crontab example ``` -10 * * * * ~/trackdirect/server/scripts/heatmapcreator.sh trackdirect.ini ~/trackdirect/htdocs/public/heatmaps -40 * * * * ~/trackdirect/server/scripts/remover.sh trackdirect.ini -0 * * * * ~/trackdirect/server/scripts/ogn_devices_install.sh trackdirect 5432 -* * * * * ~/trackdirect/server/scripts/wsserver.sh trackdirect.ini -* * * * * ~/trackdirect/server/scripts/collector.sh trackdirect.ini 0 +10 * * * * ~/trackdirect/server/scripts/heatmapcreator.sh trackdirect.ini ~/trackdirect/htdocs/public/heatmaps 2>&1 & +40 * * * * ~/trackdirect/server/scripts/remover.sh trackdirect.ini 2>&1 & +0 * * * * ~/trackdirect/server/scripts/ogn_devices_install.sh trackdirect 5432 2>&1 & +* * * * * ~/trackdirect/server/scripts/wsserver.sh trackdirect.ini 2>&1 & +* * * * * ~/trackdirect/server/scripts/collector.sh trackdirect.ini 0 2>&1 & ``` ### Server Requirements diff --git a/server/scripts/remover.sh b/server/scripts/remover.sh index 0bf96a3..c507a85 100755 --- a/server/scripts/remover.sh +++ b/server/scripts/remover.sh @@ -16,6 +16,6 @@ else export PYTHONPATH=$PYTHONPATH:$CURRENTDIR/../trackdirect:$CURRENTDIR/../../heatmap-2.2.1/ cd $CURRENTDIR/.. - python2 $CURRENTDIR/../bin/remover.py $CONFIGFILE 2>&1 & + python2 $CURRENTDIR/../bin/remover.py $CONFIGFILE exit 0 fi diff --git a/server/scripts/stationremover.sh b/server/scripts/stationremover.sh index 80aa18b..e9676ff 100755 --- a/server/scripts/stationremover.sh +++ b/server/scripts/stationremover.sh @@ -17,6 +17,6 @@ else export PYTHONPATH=$PYTHONPATH:$CURRENTDIR/../trackdirect:$CURRENTDIR/../../heatmap-2.2.1/ cd $CURRENTDIR/.. - python2 $CURRENTDIR/../bin/stationremover.py $CONFIGFILE $STATIONID 2>&1 & + python2 $CURRENTDIR/../bin/stationremover.py $CONFIGFILE $STATIONID exit 0 fi