ua1zbe
/
openwebrxplus
mirror of https://github.com/luarvique/openwebrx
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Renamed receiverdb to markers.

This commit is contained in:
Marat Fayzullin 2023-07-12 23:07:28 -04:00
parent cc2574d7cc
commit 3471e1c8c6
2 changed files with 105 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
owrx/__main__.py
View File

@ -18,7 +18,7 @@ from owrx.reporting import ReportingEngine
from owrx.version import openwebrx_version from owrx.version import openwebrx_version
from owrx.audio.queue import DecoderQueue from owrx.audio.queue import DecoderQueue
from owrx.admin import add_admin_parser, run_admin_action from owrx.admin import add_admin_parser, run_admin_action
from owrx.receiverdb import ReceiverDatabase from owrx.markers import Markers
import signal import signal
import argparse import argparse
import ssl import ssl
@ -113,7 +113,8 @@ Support and info: https://groups.io/g/openwebrx
Services.start() Services.start()
ReceiverDatabase.getSharedInstance().refresh() # Instantiate and refresh marker database
Markers.getSharedInstance().refresh()
try: try:
# This is our HTTP server # This is our HTTP server

154
owrx/receiverdb.py → owrx/markers.py
View File

@ -15,21 +15,21 @@ logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG) logger.setLevel(logging.DEBUG)
class ReceiverJSONEncoder(JSONEncoder): class MyJSONEncoder(JSONEncoder):
def default(self, obj): def default(self, obj):
return obj.toJSON() return obj.toJSON()
class ReceiverLocation(LatLngLocation): class MarkerLocation(LatLngLocation):
def __init__(self, lat: float, lon: float, attrs): def __init__(self, lat: float, lon: float, attrs):
self.attrs = attrs self.attrs = attrs
super().__init__(lat, lon) super().__init__(lat, lon)
def getId(self): def getId(self):
return re.sub(r"^.*://(.*?)(/.*)?$", r"\1", self.attrs["url"]) return self.attrs["id"]
def getMode(self): def getMode(self):
return re.sub(r"^([A-Za-z]+).*$", r"\1", self.attrs["device"]) return self.attrs["mode"]
def __dict__(self): def __dict__(self):
return self.attrs return self.attrs
@ -38,28 +38,46 @@ class ReceiverLocation(LatLngLocation):
return self.attrs return self.attrs
class ReceiverDatabase(object): class Markers(object):
sharedInstance = None sharedInstance = None
creationLock = threading.Lock() creationLock = threading.Lock()
@staticmethod @staticmethod
def getSharedInstance(): def getSharedInstance():
with ReceiverDatabase.creationLock: with Markers.creationLock:
if ReceiverDatabase.sharedInstance is None: if Markers.sharedInstance is None:
ReceiverDatabase.sharedInstance = ReceiverDatabase() Markers.sharedInstance = Markers()
return ReceiverDatabase.sharedInstance return Markers.sharedInstance
# @staticmethod
# def _getCacheFile():
# coreConfig = CoreConfig()
# return "{data_directory}/markers.json".format(data_directory=coreConfig.get_temporary_directory())
@staticmethod @staticmethod
def _getReceiversFile(): def _getMarkersFile():
coreConfig = CoreConfig() coreConfig = CoreConfig()
return "{data_directory}/receivers.json".format(data_directory=coreConfig.get_temporary_directory()) return "{data_directory}/markers.json".format(data_directory=coreConfig.get_data_directory())
def __init__(self): def __init__(self):
self.receivers = {} self.markers = {}
self.thread = None self.thread = None
# Known database files
self.fileList = [
"markers.json",
"/etc/openwebrx/markers.json",
]
# Find additional marker files in the markers.d folder
try:
markersDir = "/etc/openwebrx/markers.d"
self.fileList += [ markersDir + "/" + file
for file in os.listdir(markersDir) if file.endswith(".json")
]
except Exception:
pass
def toJSON(self): def toJSON(self):
return self.receivers return self.markers
def refresh(self): def refresh(self):
if self.thread is None: if self.thread is None:
@ -67,79 +85,73 @@ class ReceiverDatabase(object):
self.thread.start() self.thread.start()
def _refreshThread(self): def _refreshThread(self):
logger.debug("Starting receiver database refresh...") logger.debug("Starting marker database refresh...")
# No markers yet
self.markers = {}
# Load markers from local files
for file in self.fileList:
if os.path.isfile(file):
logger.debug("Loading markers from '{0}'...".format(file))
self.markers.update(self.loadMarkers(file))
# This file contains cached database # This file contains cached database
file = self._getReceiversFile() file = self._getMarkersFile()
ts = os.path.getmtime(file) if os.path.isfile(file) else 0 ts = os.path.getmtime(file) if os.path.isfile(file) else 0
# Try loading cached database from file first, unless stale # Try loading cached database from file first, unless stale
if time.time() - ts < 60*60*24: if time.time() - ts < 60*60*24:
logger.debug("Loading database from '{0}'...".format(file)) logger.debug("Loading cached markers from '{0}'...".format(file))
self.receivers = self.loadFromFile(file) self.markers.update(self.loadMarkers(file))
else: else:
self.receivers = {} # Scrape websites for data
# Scrape websites for receivers, if the list if empty
if not self.receivers:
logger.debug("Scraping KiwiSDR web site...") logger.debug("Scraping KiwiSDR web site...")
self.receivers.update(self.scrapeKiwiSDR()) self.markers.update(self.scrapeKiwiSDR())
logger.debug("Scraping WebSDR web site...") logger.debug("Scraping WebSDR web site...")
self.receivers.update(self.scrapeWebSDR()) self.markers.update(self.scrapeWebSDR())
logger.debug("Scraping OpenWebRX web site...") logger.debug("Scraping OpenWebRX web site...")
self.receivers.update(self.scrapeOWRX()) self.markers.update(self.scrapeOWRX())
#logger.debug("Scraping MWList web site...")
#self.markers.update(self.scrapeMWList())
# Save parsed data into a file # Save parsed data into a file
logger.debug("Saving {0} receivers to '{1}'...".format(len(self.receivers), file)) logger.debug("Saving {0} markers to '{1}'...".format(len(self.markers), file))
try: try:
with open(file, "w") as f: with open(file, "w") as f:
json.dump(self, f, cls=ReceiverJSONEncoder, indent=2) json.dump(self, f, cls=MyJSONEncoder, indent=2)
f.close() f.close()
except Exception as e: except Exception as e:
logger.debug("Exception: {0}".format(e)) logger.debug("Exception: {0}".format(e))
# Update map with receivers # Update map with markers
logger.debug("Updating map...") logger.debug("Updating map...")
self.updateMap() self.updateMap()
# Done # Done
logger.debug("Done refreshing receiver database.") logger.debug("Done refreshing marker database.")
self.thread = None self.thread = None
def getColor(self, type: str): def loadMarkers(self, fileName: str):
if type.startswith("KiwiSDR"): # Load markers list from JSON file
return "#800000"
elif type.startswith("WebSDR"):
return "#000080"
else:
return "#006000"
def loadFromFile(self, fileName: str = None):
# Get filename
if fileName is None:
fileName = self._getReceiversFile()
# Load receivers list from JSON file
try: try:
with open(fileName, "r") as f: with open(fileName, "r") as f:
content = f.read() db = json.load(f)
f.close() f.close()
if content:
db = json.loads(content)
except Exception as e: except Exception as e:
logger.debug("loadFromFile() exception: {0}".format(e)) logger.debug("loadMarkers() exception: {0}".format(e))
return return
# Process receivers list # Process markers list
result = {} result = {}
for key in db.keys(): for key in db.keys():
attrs = db[key] attrs = db[key]
result[key] = ReceiverLocation(attrs["lat"], attrs["lon"], attrs) result[key] = MarkerLocation(attrs["lat"], attrs["lon"], attrs)
# Done # Done
return result return result
def updateMap(self): def updateMap(self):
for r in self.receivers.values(): for r in self.markers.values():
Map.getSharedInstance().updateLocation(r.getId(), r, r.getMode()) Map.getSharedInstance().updateLocation(r.getId(), r, r.getMode())
def scrapeOWRX(self, url: str = "https://www.receiverbook.de/map"): def scrapeOWRX(self, url: str = "https://www.receiverbook.de/map"):
@ -164,8 +176,10 @@ class ReceiverDatabase(object):
dev = r["type"] + " " + r["version"] dev = r["type"] + " " + r["version"]
else: else:
dev = r["type"] dev = r["type"]
rl = ReceiverLocation(lat, lon, { rl = MarkerLocation(lat, lon, {
"type" : "feature", "type" : "feature",
"mode" : r["type"],
"id" : re.sub(r"^.*://(.*?)(/.*)?$", r"\1", r["url"]),
"lat" : lat, "lat" : lat,
"lon" : lon, "lon" : lon,
"comment" : r["label"], "comment" : r["label"],
@ -193,8 +207,10 @@ class ReceiverDatabase(object):
# Save accumulated attributes, use hostname as key # Save accumulated attributes, use hostname as key
lat = entry["lat"] lat = entry["lat"]
lon = entry["lon"] lon = entry["lon"]
rl = ReceiverLocation(lat, lon, { rl = MarkerLocation(lat, lon, {
"type" : "feature", "type" : "feature",
"mode" : "WebSDR",
"id" : re.sub(r"^.*://(.*?)(/.*)?$", r"\1", entry["url"]),
"lat" : lat, "lat" : lat,
"lon" : lon, "lon" : lon,
"comment" : entry["desc"], "comment" : entry["desc"],
@ -233,8 +249,10 @@ class ReceiverDatabase(object):
# Save accumulated attributes, use hostname as key # Save accumulated attributes, use hostname as key
lat = float(m.group(1)) lat = float(m.group(1))
lon = float(m.group(2)) lon = float(m.group(2))
rl = ReceiverLocation(lat, lon, { rl = MarkerLocation(lat, lon, {
"type" : "feature", "type" : "feature",
"mode" : "KiwiSDR",
"id" : re.sub(r"^.*://(.*?)(/.*)?$", r"\1", entry["url"]),
"lat" : lat, "lat" : lat,
"lon" : lon, "lon" : lon,
"comment" : entry["name"], "comment" : entry["name"],
@ -261,3 +279,35 @@ class ReceiverDatabase(object):
# Done # Done
return result return result
def scrapeMWList(self, url: str = "http://www.mwlist.org/shortwave_transmitter_sites.php"):
result = {}
try:
patternLoc = re.compile(r".*\['\d+',\s+'(.*?)',\s+(\S+),\s+(\S+),\s+'(\S+)',\s+(\d+)\].*")
patternUrl = re.compile(r".*<a\s+.*\s+href='(\S+locationid=)(\d+)'>.*")
for line in urllib.request.urlopen(url).readlines():
# Convert read bytes to a string
line = line.decode('utf-8')
# When we encounter a location...
m = patternLoc.match(line)
if m is not None:
rl = MarkerLocation(lat, lon, {
"type" : "feature",
"mode" : "MWList",
"id" : m.group(5),
"lat" : m.group(2),
"lon" : m.group(3),
"comment" : m.group(1) + "(" + m.group(4) + ")"
})
result[rl.getId()] = rl
else:
m = patternUrl.match(line)
if m is not None and m.group(2) in result:
result[m.group(2)].attrs["url"] = m.group(1) + m.group(2)
except Exception as e:
logger.debug("scrapeMWList() exception: {0}".format(e))
# Done
return result