Added periodic cache updates.
This commit is contained in:
Marat Fayzullin
parent
1c5688e5c1
commit
2870f43674
|
|
@ -115,10 +115,10 @@ Support and info: https://groups.io/g/openwebrx
|
|||
Services.start()
|
||||
|
||||
# Instantiate and refresh marker database
|
||||
Markers.getSharedInstance().refresh()
|
||||
Markers.start()
|
||||
|
||||
# Instantiate and refresh broadcasting schedule
|
||||
EIBI.getSharedInstance().refresh()
|
||||
EIBI.start()
|
||||
|
||||
try:
|
||||
# This is our HTTP server
|
||||
|
|
@ -142,6 +142,8 @@ Support and info: https://groups.io/g/openwebrx
|
|||
pass
|
||||
|
||||
WebSocketConnection.closeAll()
|
||||
EIBI.stop()
|
||||
Markers.stop()
|
||||
Services.stop()
|
||||
SdrService.stopAllSources()
|
||||
ReportingEngine.stopAll()
|
||||
|
|
|
|||
67
owrx/eibi.py
67
owrx/eibi.py
|
|
@ -30,53 +30,71 @@ class EIBI(object):
|
|||
EIBI.sharedInstance = EIBI()
|
||||
return EIBI.sharedInstance
|
||||
|
||||
@staticmethod
|
||||
def start():
|
||||
EIBI.getSharedInstance().startThread()
|
||||
|
||||
@staticmethod
|
||||
def stop():
|
||||
EIBI.getSharedInstance().stopThread()
|
||||
|
||||
@staticmethod
|
||||
def _getCachedScheduleFile():
|
||||
coreConfig = CoreConfig()
|
||||
return "{data_directory}/eibi.json".format(data_directory=coreConfig.get_data_directory())
|
||||
|
||||
def __init__(self):
|
||||
self.refreshPeriod = 60*60*24*30
|
||||
self.event = threading.Event()
|
||||
self.schedule = []
|
||||
self.thread = None
|
||||
|
||||
def toJSON(self):
|
||||
return self.schedule
|
||||
|
||||
def refresh(self):
|
||||
# Start the main thread
|
||||
def startThread(self):
|
||||
if self.thread is None:
|
||||
self.event.clear()
|
||||
self.thread = threading.Thread(target=self._refreshThread)
|
||||
self.thread.start()
|
||||
|
||||
# Stop the main thread
|
||||
def stopThread(self):
|
||||
if self.thread is not None:
|
||||
self.event.set()
|
||||
self.thread.join()
|
||||
|
||||
# This is the actual thread function
|
||||
def _refreshThread(self):
|
||||
logger.debug("Starting EIBI schedule refresh...")
|
||||
logger.debug("Starting EIBI main thread...")
|
||||
|
||||
# This file contains cached schedule
|
||||
file = self._getCachedScheduleFile()
|
||||
ts = os.path.getmtime(file) if os.path.isfile(file) else 0
|
||||
|
||||
# Try loading cached schedule from file first, unless stale
|
||||
if time.time() - ts < 60*60*24*30:
|
||||
if time.time() - ts < self.refreshPeriod:
|
||||
logger.debug("Loading cached schedule from '{0}'...".format(file))
|
||||
self.schedule = self.loadSchedule(file)
|
||||
else:
|
||||
# Scrape EIBI website for data
|
||||
logger.debug("Scraping EIBI web site...")
|
||||
self.schedule = self.scrape()
|
||||
# Save parsed data into a file
|
||||
logger.debug("Saving {0} schedule entries to '{1}'...".format(len(self.schedule), file))
|
||||
try:
|
||||
with open(file, "w") as f:
|
||||
json.dump(self, f, cls=MyJSONEncoder, indent=2)
|
||||
f.close()
|
||||
except Exception as e:
|
||||
logger.debug("Exception: {0}".format(e))
|
||||
self.schedule = self.updateSchedule()
|
||||
|
||||
while not self.event.is_set():
|
||||
# Sleep until it is time to update schedule
|
||||
self.event.wait(self.refreshPeriod)
|
||||
# If not terminated yet...
|
||||
if not self.event.is_set():
|
||||
# Update schedule
|
||||
logger.debug("Refreshing schedule...")
|
||||
self.schedule = self.updateSchedule()
|
||||
|
||||
# Done
|
||||
logger.debug("Done refreshing schedule.")
|
||||
logger.debug("Stopped EIBI main thread.")
|
||||
self.thread = None
|
||||
|
||||
# Load schedule from a given JSON file
|
||||
def loadSchedule(self, fileName: str):
|
||||
# Load schedule from JSON file
|
||||
try:
|
||||
with open(fileName, "r") as f:
|
||||
result = json.load(f)
|
||||
|
|
@ -87,6 +105,23 @@ class EIBI(object):
|
|||
# Done
|
||||
return result
|
||||
|
||||
# Update schedule
|
||||
def updateSchedule(self):
|
||||
# Scrape EIBI database file
|
||||
logger.debug("Scraping EIBI website...")
|
||||
file = self._getCachedScheduleFile()
|
||||
schedule = self.scrape()
|
||||
# Save parsed data into a file
|
||||
logger.debug("Saving {0} schedule entries to '{1}'...".format(len(schedule), file))
|
||||
try:
|
||||
with open(file, "w") as f:
|
||||
json.dump(schedule, f, cls=MyJSONEncoder, indent=2)
|
||||
f.close()
|
||||
except Exception as e:
|
||||
logger.debug("updateSchedule() exception: {0}".format(e))
|
||||
# Done
|
||||
return schedule
|
||||
|
||||
def findBySource(self, src: str):
|
||||
# Get entries active at the current time
|
||||
now = datetime.utcnow()
|
||||
|
|
|
|||
125
owrx/markers.py
125
owrx/markers.py
|
|
@ -49,12 +49,22 @@ class Markers(object):
|
|||
Markers.sharedInstance = Markers()
|
||||
return Markers.sharedInstance
|
||||
|
||||
@staticmethod
|
||||
def start():
|
||||
Markers.getSharedInstance().startThread()
|
||||
|
||||
@staticmethod
|
||||
def stop():
|
||||
Markers.getSharedInstance().stopThread()
|
||||
|
||||
@staticmethod
|
||||
def _getCachedMarkersFile():
|
||||
coreConfig = CoreConfig()
|
||||
return "{data_directory}/markers.json".format(data_directory=coreConfig.get_data_directory())
|
||||
|
||||
def __init__(self):
|
||||
self.refreshPeriod = 60*60*24
|
||||
self.event = threading.Event()
|
||||
self.markers = {}
|
||||
self.thread = None
|
||||
# Known database files
|
||||
|
|
@ -74,63 +84,65 @@ class Markers(object):
|
|||
def toJSON(self):
|
||||
return self.markers
|
||||
|
||||
def refresh(self):
|
||||
# Start the main thread
|
||||
def startThread(self):
|
||||
if self.thread is None:
|
||||
self.event.clear()
|
||||
self.thread = threading.Thread(target=self._refreshThread)
|
||||
self.thread.start()
|
||||
|
||||
# Stop the main thread
|
||||
def stopThread(self):
|
||||
if self.thread is not None:
|
||||
self.event.set()
|
||||
self.thread.join()
|
||||
|
||||
# This is the actual thread function
|
||||
def _refreshThread(self):
|
||||
logger.debug("Starting marker database refresh...")
|
||||
logger.debug("Starting marker database thread...")
|
||||
|
||||
# No markers yet
|
||||
self.markers = {}
|
||||
|
||||
# This file contains cached database
|
||||
file = self._getCachedMarkersFile()
|
||||
ts = os.path.getmtime(file) if os.path.isfile(file) else 0
|
||||
|
||||
# Try loading cached database from file first, unless stale
|
||||
if time.time() - ts < 60*60*24:
|
||||
logger.debug("Loading cached markers from '{0}'...".format(file))
|
||||
self.markers.update(self.loadMarkers(file))
|
||||
else:
|
||||
# Scrape websites for data
|
||||
cache = {}
|
||||
logger.debug("Scraping KiwiSDR web site...")
|
||||
cache.update(self.scrapeKiwiSDR())
|
||||
logger.debug("Scraping WebSDR web site...")
|
||||
cache.update(self.scrapeWebSDR())
|
||||
logger.debug("Scraping OpenWebRX web site...")
|
||||
cache.update(self.scrapeOWRX())
|
||||
# Save parsed data into a file
|
||||
logger.debug("Saving {0} markers to '{1}'...".format(len(self.markers), file))
|
||||
try:
|
||||
with open(file, "w") as f:
|
||||
json.dump(cache, f, cls=MyJSONEncoder, indent=2)
|
||||
f.close()
|
||||
except Exception as e:
|
||||
logger.debug("Exception: {0}".format(e))
|
||||
# Add scraped data to the database
|
||||
self.markers.update(cache)
|
||||
|
||||
# Load markers from the EIBI database
|
||||
logger.debug("Loading EIBI transmitter locations...")
|
||||
self.markers.update(self.loadEIBI())
|
||||
|
||||
# Load markers from local files
|
||||
for file in self.fileList:
|
||||
if os.path.isfile(file):
|
||||
logger.debug("Loading markers from '{0}'...".format(file))
|
||||
self.markers.update(self.loadMarkers(file))
|
||||
|
||||
# Update map with markers
|
||||
logger.debug("Updating map...")
|
||||
self.updateMap()
|
||||
# Load markers from the EIBI database
|
||||
logger.debug("Loading EIBI transmitter locations...")
|
||||
self.markers.update(self.loadEIBI())
|
||||
|
||||
# Done
|
||||
logger.debug("Done refreshing marker database.")
|
||||
# This file contains cached database
|
||||
file = self._getCachedMarkersFile()
|
||||
ts = os.path.getmtime(file) if os.path.isfile(file) else 0
|
||||
|
||||
# Try loading cached database from file first, unless stale
|
||||
if time.time() - ts < self.refreshPeriod:
|
||||
logger.debug("Loading cached markers from '{0}'...".format(file))
|
||||
self.markers.update(self.loadMarkers(file))
|
||||
else:
|
||||
# Add scraped data to the database
|
||||
self.markers.update(self.updateCache())
|
||||
|
||||
while not self.event.is_set():
|
||||
# Update map with markers
|
||||
logger.debug("Updating map...")
|
||||
self.updateMap()
|
||||
# Sleep until it is time to update schedule
|
||||
self.event.wait(self.refreshPeriod)
|
||||
# If not terminated yet...
|
||||
if not self.event.is_set():
|
||||
# Scrape data, updating cache
|
||||
logger.debug("Refreshing marker database...")
|
||||
self.markers.update(self.updateCache())
|
||||
|
||||
# Done with the thread
|
||||
logger.debug("Stopped marker database thread.")
|
||||
self.thread = None
|
||||
|
||||
# Load markers from a given file
|
||||
def loadMarkers(self, fileName: str):
|
||||
# Load markers list from JSON file
|
||||
try:
|
||||
|
|
@ -150,6 +162,37 @@ class Markers(object):
|
|||
# Done
|
||||
return result
|
||||
|
||||
# Update markers on the map
|
||||
def updateMap(self):
|
||||
for r in self.markers.values():
|
||||
Map.getSharedInstance().updateLocation(r.getId(), r, r.getMode(), permanent=True)
|
||||
|
||||
# Scrape online databases, updating cache file
|
||||
def updateCache(self):
|
||||
# Scrape websites for data
|
||||
file = self._getCachedMarkersFile()
|
||||
cache = {}
|
||||
logger.debug("Scraping KiwiSDR website...")
|
||||
cache.update(self.scrapeKiwiSDR())
|
||||
logger.debug("Scraping WebSDR website...")
|
||||
cache.update(self.scrapeWebSDR())
|
||||
logger.debug("Scraping OpenWebRX website...")
|
||||
cache.update(self.scrapeOWRX())
|
||||
# Save parsed data into a file
|
||||
logger.debug("Saving {0} markers to '{1}'...".format(len(cache), file))
|
||||
try:
|
||||
with open(file, "w") as f:
|
||||
json.dump(cache, f, cls=MyJSONEncoder, indent=2)
|
||||
f.close()
|
||||
except Exception as e:
|
||||
logger.debug("updateCache() exception: {0}".format(e))
|
||||
# Done
|
||||
return cache
|
||||
|
||||
#
|
||||
# Following functions scrape data from websites and internal databases
|
||||
#
|
||||
|
||||
def loadEIBI(self):
|
||||
#url = "https://www.short-wave.info/index.php?txsite="
|
||||
url = "https://www.google.com/search?q="
|
||||
|
|
@ -169,10 +212,6 @@ class Markers(object):
|
|||
# Done
|
||||
return result
|
||||
|
||||
def updateMap(self):
|
||||
for r in self.markers.values():
|
||||
Map.getSharedInstance().updateLocation(r.getId(), r, r.getMode(), permanent=True)
|
||||
|
||||
def scrapeOWRX(self, url: str = "https://www.receiverbook.de/map"):
|
||||
patternJson = re.compile(r"^\s*var\s+receivers\s+=\s+(\[.*\]);\s*$")
|
||||
result = {}
|
||||
|
|
|
|||
Loading…
Reference in New Issue