From 2870f436749176f886debb16ebc6614b6c2c7cbb Mon Sep 17 00:00:00 2001 From: Marat Fayzullin Date: Sat, 15 Jul 2023 12:22:07 -0400 Subject: [PATCH] Added periodic cache updates. --- owrx/__main__.py | 6 ++- owrx/eibi.py | 67 +++++++++++++++++++------ owrx/markers.py | 125 +++++++++++++++++++++++++++++++---------------- 3 files changed, 137 insertions(+), 61 deletions(-) diff --git a/owrx/__main__.py b/owrx/__main__.py index 282411c9..e19b0c52 100644 --- a/owrx/__main__.py +++ b/owrx/__main__.py @@ -115,10 +115,10 @@ Support and info: https://groups.io/g/openwebrx Services.start() # Instantiate and refresh marker database - Markers.getSharedInstance().refresh() + Markers.start() # Instantiate and refresh broadcasting schedule - EIBI.getSharedInstance().refresh() + EIBI.start() try: # This is our HTTP server @@ -142,6 +142,8 @@ Support and info: https://groups.io/g/openwebrx pass WebSocketConnection.closeAll() + EIBI.stop() + Markers.stop() Services.stop() SdrService.stopAllSources() ReportingEngine.stopAll() diff --git a/owrx/eibi.py b/owrx/eibi.py index 094b25fc..475cae63 100644 --- a/owrx/eibi.py +++ b/owrx/eibi.py @@ -30,53 +30,71 @@ class EIBI(object): EIBI.sharedInstance = EIBI() return EIBI.sharedInstance + @staticmethod + def start(): + EIBI.getSharedInstance().startThread() + + @staticmethod + def stop(): + EIBI.getSharedInstance().stopThread() + @staticmethod def _getCachedScheduleFile(): coreConfig = CoreConfig() return "{data_directory}/eibi.json".format(data_directory=coreConfig.get_data_directory()) def __init__(self): + self.refreshPeriod = 60*60*24*30 + self.event = threading.Event() self.schedule = [] self.thread = None def toJSON(self): return self.schedule - def refresh(self): + # Start the main thread + def startThread(self): if self.thread is None: + self.event.clear() self.thread = threading.Thread(target=self._refreshThread) self.thread.start() + # Stop the main thread + def stopThread(self): + if self.thread is not None: + self.event.set() + self.thread.join() + + # This is the actual thread function def _refreshThread(self): - logger.debug("Starting EIBI schedule refresh...") + logger.debug("Starting EIBI main thread...") # This file contains cached schedule file = self._getCachedScheduleFile() ts = os.path.getmtime(file) if os.path.isfile(file) else 0 # Try loading cached schedule from file first, unless stale - if time.time() - ts < 60*60*24*30: + if time.time() - ts < self.refreshPeriod: logger.debug("Loading cached schedule from '{0}'...".format(file)) self.schedule = self.loadSchedule(file) else: - # Scrape EIBI website for data - logger.debug("Scraping EIBI web site...") - self.schedule = self.scrape() - # Save parsed data into a file - logger.debug("Saving {0} schedule entries to '{1}'...".format(len(self.schedule), file)) - try: - with open(file, "w") as f: - json.dump(self, f, cls=MyJSONEncoder, indent=2) - f.close() - except Exception as e: - logger.debug("Exception: {0}".format(e)) + self.schedule = self.updateSchedule() + + while not self.event.is_set(): + # Sleep until it is time to update schedule + self.event.wait(self.refreshPeriod) + # If not terminated yet... + if not self.event.is_set(): + # Update schedule + logger.debug("Refreshing schedule...") + self.schedule = self.updateSchedule() # Done - logger.debug("Done refreshing schedule.") + logger.debug("Stopped EIBI main thread.") self.thread = None + # Load schedule from a given JSON file def loadSchedule(self, fileName: str): - # Load schedule from JSON file try: with open(fileName, "r") as f: result = json.load(f) @@ -87,6 +105,23 @@ class EIBI(object): # Done return result + # Update schedule + def updateSchedule(self): + # Scrape EIBI database file + logger.debug("Scraping EIBI website...") + file = self._getCachedScheduleFile() + schedule = self.scrape() + # Save parsed data into a file + logger.debug("Saving {0} schedule entries to '{1}'...".format(len(schedule), file)) + try: + with open(file, "w") as f: + json.dump(schedule, f, cls=MyJSONEncoder, indent=2) + f.close() + except Exception as e: + logger.debug("updateSchedule() exception: {0}".format(e)) + # Done + return schedule + def findBySource(self, src: str): # Get entries active at the current time now = datetime.utcnow() diff --git a/owrx/markers.py b/owrx/markers.py index 1edde74c..baf4afcb 100644 --- a/owrx/markers.py +++ b/owrx/markers.py @@ -49,12 +49,22 @@ class Markers(object): Markers.sharedInstance = Markers() return Markers.sharedInstance + @staticmethod + def start(): + Markers.getSharedInstance().startThread() + + @staticmethod + def stop(): + Markers.getSharedInstance().stopThread() + @staticmethod def _getCachedMarkersFile(): coreConfig = CoreConfig() return "{data_directory}/markers.json".format(data_directory=coreConfig.get_data_directory()) def __init__(self): + self.refreshPeriod = 60*60*24 + self.event = threading.Event() self.markers = {} self.thread = None # Known database files @@ -74,63 +84,65 @@ class Markers(object): def toJSON(self): return self.markers - def refresh(self): + # Start the main thread + def startThread(self): if self.thread is None: + self.event.clear() self.thread = threading.Thread(target=self._refreshThread) self.thread.start() + # Stop the main thread + def stopThread(self): + if self.thread is not None: + self.event.set() + self.thread.join() + + # This is the actual thread function def _refreshThread(self): - logger.debug("Starting marker database refresh...") + logger.debug("Starting marker database thread...") # No markers yet self.markers = {} - # This file contains cached database - file = self._getCachedMarkersFile() - ts = os.path.getmtime(file) if os.path.isfile(file) else 0 - - # Try loading cached database from file first, unless stale - if time.time() - ts < 60*60*24: - logger.debug("Loading cached markers from '{0}'...".format(file)) - self.markers.update(self.loadMarkers(file)) - else: - # Scrape websites for data - cache = {} - logger.debug("Scraping KiwiSDR web site...") - cache.update(self.scrapeKiwiSDR()) - logger.debug("Scraping WebSDR web site...") - cache.update(self.scrapeWebSDR()) - logger.debug("Scraping OpenWebRX web site...") - cache.update(self.scrapeOWRX()) - # Save parsed data into a file - logger.debug("Saving {0} markers to '{1}'...".format(len(self.markers), file)) - try: - with open(file, "w") as f: - json.dump(cache, f, cls=MyJSONEncoder, indent=2) - f.close() - except Exception as e: - logger.debug("Exception: {0}".format(e)) - # Add scraped data to the database - self.markers.update(cache) - - # Load markers from the EIBI database - logger.debug("Loading EIBI transmitter locations...") - self.markers.update(self.loadEIBI()) - # Load markers from local files for file in self.fileList: if os.path.isfile(file): logger.debug("Loading markers from '{0}'...".format(file)) self.markers.update(self.loadMarkers(file)) - # Update map with markers - logger.debug("Updating map...") - self.updateMap() + # Load markers from the EIBI database + logger.debug("Loading EIBI transmitter locations...") + self.markers.update(self.loadEIBI()) - # Done - logger.debug("Done refreshing marker database.") + # This file contains cached database + file = self._getCachedMarkersFile() + ts = os.path.getmtime(file) if os.path.isfile(file) else 0 + + # Try loading cached database from file first, unless stale + if time.time() - ts < self.refreshPeriod: + logger.debug("Loading cached markers from '{0}'...".format(file)) + self.markers.update(self.loadMarkers(file)) + else: + # Add scraped data to the database + self.markers.update(self.updateCache()) + + while not self.event.is_set(): + # Update map with markers + logger.debug("Updating map...") + self.updateMap() + # Sleep until it is time to update schedule + self.event.wait(self.refreshPeriod) + # If not terminated yet... + if not self.event.is_set(): + # Scrape data, updating cache + logger.debug("Refreshing marker database...") + self.markers.update(self.updateCache()) + + # Done with the thread + logger.debug("Stopped marker database thread.") self.thread = None + # Load markers from a given file def loadMarkers(self, fileName: str): # Load markers list from JSON file try: @@ -150,6 +162,37 @@ class Markers(object): # Done return result + # Update markers on the map + def updateMap(self): + for r in self.markers.values(): + Map.getSharedInstance().updateLocation(r.getId(), r, r.getMode(), permanent=True) + + # Scrape online databases, updating cache file + def updateCache(self): + # Scrape websites for data + file = self._getCachedMarkersFile() + cache = {} + logger.debug("Scraping KiwiSDR website...") + cache.update(self.scrapeKiwiSDR()) + logger.debug("Scraping WebSDR website...") + cache.update(self.scrapeWebSDR()) + logger.debug("Scraping OpenWebRX website...") + cache.update(self.scrapeOWRX()) + # Save parsed data into a file + logger.debug("Saving {0} markers to '{1}'...".format(len(cache), file)) + try: + with open(file, "w") as f: + json.dump(cache, f, cls=MyJSONEncoder, indent=2) + f.close() + except Exception as e: + logger.debug("updateCache() exception: {0}".format(e)) + # Done + return cache + + # + # Following functions scrape data from websites and internal databases + # + def loadEIBI(self): #url = "https://www.short-wave.info/index.php?txsite=" url = "https://www.google.com/search?q=" @@ -169,10 +212,6 @@ class Markers(object): # Done return result - def updateMap(self): - for r in self.markers.values(): - Map.getSharedInstance().updateLocation(r.getId(), r, r.getMode(), permanent=True) - def scrapeOWRX(self, url: str = "https://www.receiverbook.de/map"): patternJson = re.compile(r"^\s*var\s+receivers\s+=\s+(\[.*\]);\s*$") result = {}