ua1zbe
/
openwebrxplus
mirror of https://github.com/luarvique/openwebrx
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'luarvique:master' into master

This commit is contained in:
Stanislav Lechev [0xAF] 2023-07-24 01:30:02 +03:00 committed by GitHub
parent 59e3cb78fd 41b655df82
commit 3871679ed9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 510 additions and 202 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
CHANGELOG.md
View File

@ -1,3 +1,10 @@
**1.2.24**
- Added support for EIBI shortwave schedules.
- Schedules updated monthly from the EIBI website.
- Map shows currently active transmitters, with 1-hour schedules.
- You can instantly tune by clicking on a schedule entry.
- Your current SDR profile must contain the clicked frequency.
**1.2.23**
- Added OpenWebRX, WebSDR, and KiwiSDR locations to the map.
- Added periodic updates of online SDR locations from the web.

10
debian/changelog vendored
View File

@ -1,3 +1,13 @@
openwebrx (1.2.24) bullseye jammy; urgency=low
* Added support for EIBI shortwave schedules.
* Schedules updated monthly from the EIBI website.
* Map shows currently active transmitters, with 1-hour schedules.
* You can instantly tune by clicking on a schedule entry.
* Your current SDR profile must contain the clicked frequency.
-- Marat Fayzullin <luarvique@gmail.com> Sat, 22 Jul 2023 16:12:00 +0000
openwebrx (1.2.23) bullseye jammy; urgency=low
* Added OpenWebRX, WebSDR, and KiwiSDR locations to the map.

2
htdocs/lib/MarkerManager.js
View File

@ -10,7 +10,7 @@ function MarkerManager() {
this.colors = {
'KiwiSDR' : '#800000',
'WebSDR' : '#000080',
'OpenWebRX' : '#006000'
'OpenWebRX' : '#004000'
};
// Symbols used for features

3
htdocs/map.html
View File

@ -22,9 +22,10 @@
<option value="off">Off</option>
</select>
<div class="content"></div>
<h3>Features</h3>
<h3 style="margin: 20px 0 0 0;">Features</h3>
<ul class="features">
</ul>
<h3 id="openwebrx-clock-utc" style="margin: 20px 0 0 0;">00:00 UTC</h3>
</div>
</body>
</html>

55
htdocs/map.js
View File

@ -44,6 +44,9 @@ $(function(){
// marker manager
var markmanager = null;
// clock
var clock = new Clock($("#openwebrx-clock-utc"));
var colorKeys = {};
var colorScale = chroma.scale(['red', 'blue', 'green']).mode('hsl');
var getColor = function(id){
@ -206,14 +209,14 @@ $(function(){
}, options));
// Get attributes
//features have no expiration date
//marker.lastseen = update.lastseen;
marker.lastseen = update.lastseen;
marker.mode = update.mode;
marker.url = update.location.url;
marker.comment = update.location.comment;
marker.altitude = update.location.altitude;
marker.device = update.location.device;
marker.antenna = update.location.antenna;
marker.schedule = update.location.schedule;
if (expectedCallsign && expectedCallsign == update.callsign) {
map.panTo(pos);
@ -457,7 +460,7 @@ $(function(){
return '<a target="callsign_info" href="' +
url.replaceAll('{}', callsign.replace(new RegExp('-.*$'), '')) +
'">' + callsign + '</a>';
};
}
var distanceKm = function(p1, p2) {
// Earth radius in km
@ -520,8 +523,8 @@ $(function(){
}
var makeListItem = function(name, value) {
return '<div style="border-bottom:1px dotted;">'
+ '<span>' + name + '</span>'
return '<div style="border-bottom:1px dotted;white-space:nowrap;">'
+ '<span>' + name + '&nbsp;&nbsp;&nbsp;&nbsp;</span>'
+ '<span style="float:right;">' + value + '</span>'
+ '</div>';
}
@ -664,6 +667,7 @@ $(function(){
var marker = markers[name];
var commentString = "";
var detailsString = "";
var scheduleString = "";
var nameString = "";
var distance = "";
@ -692,17 +696,42 @@ $(function(){
detailsString += makeListItem('Antenna', truncate(marker.antenna, 24));
}
if (marker.schedule) {
for (var j=0 ; j<marker.schedule.length ; ++j) {
var freq = marker.schedule[j].freq;
var mode = marker.schedule[j].mode;
var tune = mode === 'cw'? freq - 800
: mode === 'fax'? freq - 1500
: mode === 'rtty450'? freq - 1000
: freq;
var name = ("0000" + marker.schedule[j].time1).slice(-4)
+ "&#8209;" + ("0000" + marker.schedule[j].time2).slice(-4)
+ "&nbsp;&nbsp;" + marker.schedule[j].name;
freq = '<a target="openwebrx-rx" href="/#freq=' + tune
+ ',mod=' + (mode? mode : 'am') + '">'
+ Math.round(marker.schedule[j].freq/1000) + 'kHz</a>';
scheduleString += makeListItem(name, freq);
}
}
if (detailsString.length > 0) {
detailsString = '<p>' + makeListTitle('Details') + detailsString + '</p>';
}
if (scheduleString.length > 0) {
scheduleString = '<p>' + makeListTitle('Schedule') + scheduleString + '</p>';
}
if (receiverMarker) {
distance = " at " + distanceKm(receiverMarker.position, marker.position) + " km";
}
infowindow.setContent(
'<h3>' + nameString + distance + '</h3>' +
commentString + detailsString
commentString + detailsString + scheduleString
);
infowindow.open(map, marker);
@ -754,15 +783,13 @@ $(function(){
m.setOptions(getRectangleOpacityOptions(m.lastseen));
});
$.each(markers, function(callsign, m) {
if (m.lastseen) {
var age = now - m.lastseen;
if (age > retention_time) {
delete markers[callsign];
m.setMap();
return;
}
m.setOptions(getMarkerOpacityOptions(m.lastseen));
var age = now - m.lastseen;
if (age > retention_time) {
delete markers[callsign];
m.setMap();
return;
}
m.setOptions(getMarkerOpacityOptions(m.lastseen));
});
}, 1000);

3
htdocs/openwebrx.js
View File

@ -1577,6 +1577,9 @@ var bookmarks;
var audioEngine;
function openwebrx_init() {
// Name used by map links to tune receiver
frames.name = 'openwebrx-rx';
audioEngine = new AudioEngine(audio_buffer_maximal_length_sec, audioReporter);
var $overlay = $('#openwebrx-autoplay-overlay');
$overlay.on('click', function(){

5
owrx/__main__.py
View File

@ -19,7 +19,6 @@ from owrx.version import openwebrx_version
from owrx.audio.queue import DecoderQueue
from owrx.admin import add_admin_parser, run_admin_action
from owrx.markers import Markers
from owrx.eibi import EIBI
import signal
import argparse
import ssl
@ -117,9 +116,6 @@ Support and info: https://groups.io/g/openwebrx
# Instantiate and refresh marker database
Markers.start()
# Instantiate and refresh broadcasting schedule
EIBI.start()
try:
# This is our HTTP server
server = ThreadedHttpServer(("0.0.0.0", coreConfig.get_web_port()), RequestHandler)
@ -142,7 +138,6 @@ Support and info: https://groups.io/g/openwebrx
pass
WebSocketConnection.closeAll()
EIBI.stop()
Markers.stop()
Services.stop()
SdrService.stopAllSources()

1
owrx/controllers/assets.py
View File

@ -145,6 +145,7 @@ class CompiledAssetsController(GzipMixin, ModificationAwareController):
"lib/jquery-3.2.1.min.js",
"lib/chroma.min.js",
"lib/Header.js",
"lib/Clock.js",
"map.js",
],
"settings.js": [

438
owrx/eibi.py
View File

@ -14,11 +14,6 @@ logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
class MyJSONEncoder(JSONEncoder):
def default(self, obj):
return obj.toJSON()
class EIBI(object):
sharedInstance = None
creationLock = threading.Lock()
@ -44,185 +39,362 @@ class EIBI(object):
return "{data_directory}/eibi.json".format(data_directory=coreConfig.get_data_directory())
def __init__(self):
self.patternCSV = re.compile(r"^([\d\.]+);(\d\d\d\d)-(\d\d\d\d);(\S*);(\S+);(.*);(.*);(.*);(.*);(\d+);(.*);(.*)$")
self.patternDays = re.compile(r"^(.*)(Mo|Tu|We|Th|Fr|Sa|Su)-(Mo|Tu|We|Th|Fr|Sa|Su)(.*)$")
self.refreshPeriod = 60*60*24*30
self.event = threading.Event()
self.lock = threading.Lock()
self.schedule = []
self.thread = None
def toJSON(self):
return self.schedule
# Start the main thread
def startThread(self):
if self.thread is None:
self.event.clear()
self.thread = threading.Thread(target=self._refreshThread)
self.thread.start()
# Stop the main thread
def stopThread(self):
if self.thread is not None:
self.event.set()
self.thread.join()
# This is the actual thread function
def _refreshThread(self):
logger.debug("Starting EIBI main thread...")
# Load cached schedule or refresh it from the web
def refresh(self):
# This file contains cached schedule
file = self._getCachedScheduleFile()
ts = os.path.getmtime(file) if os.path.isfile(file) else 0
# Try loading cached schedule from file first, unless stale
if time.time() - ts < self.refreshPeriod:
logger.debug("Loading cached schedule from '{0}'...".format(file))
self.schedule = self.loadSchedule(file)
else:
self.schedule = self.updateSchedule()
with self.lock:
# If cached schedule is stale...
if time.time() - ts >= self.refreshPeriod:
# Load updated schedule from the web
schedule = self.updateSchedule()
# Only update current schedule if updated from the web
if schedule:
self.schedule = schedule
while not self.event.is_set():
# Sleep until it is time to update schedule
self.event.wait(self.refreshPeriod)
# If not terminated yet...
if not self.event.is_set():
# Update schedule
logger.debug("Refreshing schedule...")
self.schedule = self.updateSchedule()
# If no current schedule, load it from cached file
if not self.schedule:
self.schedule = self.loadSchedule(file)
# Done
logger.debug("Stopped EIBI main thread.")
self.thread = None
# Save schedule to a given JSON file
def saveSchedule(self, file: str, schedule):
logger.debug("Saving {0} schedule entries to '{1}'...".format(len(schedule), file))
try:
with open(file, "w") as f:
json.dump(schedule, f, indent=2)
f.close()
except Exception as e:
logger.debug("saveSchedule() exception: {0}".format(e))
# Load schedule from a given JSON file
def loadSchedule(self, fileName: str):
def loadSchedule(self, file: str):
logger.debug("Loading schedule from '{0}'...".format(file))
try:
with open(fileName, "r") as f:
with open(file, "r") as f:
result = json.load(f)
f.close()
except Exception as e:
logger.debug("loadSchedule() exception: {0}".format(e))
result = []
# Done
logger.debug("Loaded {0} entries from '{1}'...".format(len(result), file))
return result
# Update schedule
def updateSchedule(self):
# Scrape EIBI database file
logger.debug("Scraping EIBI website...")
# Load EIBI database file from the web
file = self._getCachedScheduleFile()
schedule = self.scrape()
schedule = self.loadFromWeb()
# Save parsed data into a file
logger.debug("Saving {0} schedule entries to '{1}'...".format(len(schedule), file))
try:
with open(file, "w") as f:
json.dump(schedule, f, cls=MyJSONEncoder, indent=2)
f.close()
except Exception as e:
logger.debug("updateSchedule() exception: {0}".format(e))
if schedule:
self.saveSchedule(file, schedule)
# Done
return schedule
# Find all current broadcasts for a given source
def findBySource(self, src: str):
# Get entries active at the current time
now = datetime.utcnow()
now = now.hour * 100 + now.minute
result = []
# Search for entries originating from given source at current time
for entry in self.schedule:
if entry["time1"] <= now and entry["time2"] > now:
if entry["itu"] + entry["src"] == src:
result.append(entry)
with self.lock:
for entry in self.schedule:
if entry["time1"] <= now and entry["time2"] > now:
if entry["itu"] + entry["src"] == src:
result.append(entry)
# Done
return result
# Find all current broadcasts for a given frequency range
def findCurrent(self, freq1: int, freq2: int):
# Get entries active at the current time
now = datetime.utcnow()
now = now.hour * 100 + now.minute
return self.find(freq1, freq2, now, now)
# Find all broadcasts for given frequency and time ranges
def find(self, freq1: int, freq2: int, time1: int, time2: int):
result = []
# Search for entries within given frequency and time ranges
for entry in self.schedule:
f = entry["freq"]
if f >= freq1 and f <= freq2:
if entry["time1"] <= time2 and entry["time2"] > time1:
result.append(entry)
with self.lock:
for entry in self.schedule:
f = entry["freq"]
if f >= freq1 and f <= freq2:
if entry["time1"] <= time2 and entry["time2"] > time1:
result.append(entry)
# Done
return result
def convertDays(self, days: str):
# Replace day names with digits, remove commas
days = re.sub("Mo", "1", days)
days = re.sub("Tu", "2", days)
days = re.sub("We", "3", days)
days = re.sub("Th", "4", days)
days = re.sub("Fr", "5", days)
days = re.sub("Sa", "6", days)
days = re.sub("Su", "7", days)
days = re.sub(r"[^0-9\-]", "", days)
# Empty strings mean every day of the week
if days == "":
return "1234567"
# Parse input string
span = True
curr = 1
out = ""
for j in range(len(days)):
if days[j] == "-":
span = True
else:
next = int(days[j])
while curr < next:
out += str(curr) if span else "."
curr += 1
span = False
# Add final days
while curr < 8:
out += str(curr) if span else "."
curr += 1
# Done
return out
# Create list of currently broadcasting locations
def currentTransmitters(self, hours: int = 1):
# Get entries active at the current time + 1 hour
now = datetime.utcnow()
day = now.weekday()
date = now.year * 10000 + now.month * 100 + now.day
t1 = now.hour * 100 + now.minute
t2 = t1 + hours * 100
result = {}
# Search for current entries
with self.lock:
for entry in self.schedule:
# Check if entry is currently active
entryActive = (
entry["days"][day] != "."
and (entry["date1"] == 0 or entry["date1"] <= date)
and (entry["date2"] == 0 or entry["date2"] >= date)
)
# Check the hours, rolling over to the next day
if entryActive:
e1 = entry["time1"]
e2 = entry["time2"]
e2 = e2 if e2 > e1 else e2 + 2400
entryActive = e1 < t2 and e2 > t1
# For evere currently active schedule entry...
if entryActive:
src = entry["itu"] + entry["src"]
# Find all matching transmitter locations
for loc in EIBI_Locations:
if loc["code"] == src:
# Add location to the result
name = loc["name"]
if name not in result:
result[name] = loc.copy()
result[name]["schedule"] = []
# Add schedule entry to the location
result[name]["schedule"].append(entry)
def scrape(self, url: str = "http://www.eibispace.de/dx/sked-a23.csv"):
# Done
return result
def convertDate(self, date: str):
# No-date is a common case
if date == "":
return 0
# Remove last-seen data
date = re.sub(r"\[.*\]", "", date)
# Match day/month
m = re.match(r"^(\d\d)(\d\d)$", date)
if m is None:
return 0
else:
now = datetime.utcnow()
month = int(m.group(2))
day = int(m.group(1))
year = (
now.year + 1 if (now.month >= 11) and (month < now.month) else
now.year - 1 if (now.month <= 3) and (month > now.month) else
now.year
)
return year * 10000 + month * 100 + day
def convertDays(self, days: str):
# Look up and process special cases
if days in EIBI_SpecialDays:
return EIBI_SpecialDays[days]
# Start with empty result
result = [ ".", ".", ".", ".", ".", ".", "."]
# Extract day spans
m = self.patternDays.match(days)
if m is not None:
x = EIBI_Days[m.group(2)]
y = EIBI_Days[m.group(3)]
result[y - 1] = str(y)
while x != y:
result[x - 1] = str(x)
x = x + 1 if x < 7 else 1
# Remove extracted span
days = m.group(1) + m.group(4)
# Extract singular days
for day in EIBI_Days.keys():
if day in days:
x = EIBI_Days[day]
result[x - 1] = str(x)
# Done
return "".join(result)
def loadFromWeb(self, url: str = "http://www.eibispace.de/dx/sked-{0}.csv"):
# Figure out CSV file name based on the current date
# SUMMER: Apr - Oct - sked-aNN.csv
# WINTER: Nov - Mar - sked-bNN.csv
now = datetime.utcnow()
url = url.format(
("a" if now.month >= 4 and now.month <= 10 else "b") +
str((now.year if now.month >= 4 else now.year - 1) % 100)
)
# Fetch and parse CSV file
result = []
try:
# This is out CSV pattern
pattern = re.compile(r"^([\d\.]+);(\d\d\d\d)-(\d\d\d\d);(\S*);(\S+);(.*);(.*);(.*);(.*);(.*);(.*);(.*)$")
logger.debug("Scraping '{0}'...".format(url))
for line in urllib.request.urlopen(url).readlines():
# Convert read bytes to a string
line = line.decode('cp1252').rstrip()
# When we encounter a location...
m = pattern.match(line)
m = self.patternCSV.match(line)
if m is not None:
freq = int(float(m.group(1)) * 1000)
days = m.group(4)
name = m.group(6).lower()
lang = m.group(7)
trgt = m.group(8)
# Guess modulation, default to AM
mode = (
"hfdl" if lang == "-HF" else
"rtty450" if lang == "-TY" else
"cw" if lang == "-CW" else
"usb" if days == "USB" else
"lsb" if days == "LSB" else
"hfdl" if "hfdl" in name else # HFDL
"drm" if "digital" in name else # DRM
"fax" if " fax" in name else # Weather FAX
"rtty450" if "rtty" in name else # Weather RTTY
"usb" if "volmet" in name else # Weather
"usb" if "cross " in name else # Weather
"usb" if " ldoc" in name else # Aircraft
"usb" if " car-" in name else # Aircraft
"usb" if " nat-" in name else # Aircraft
"usb" if " usb" in name else
"usb" if "fsk" in name else
"usb" if freq < 7000000 else # Services
"am")
# Convert language code to language
if lang in EIBI_Languages:
lang = EIBI_Languages[lang]["name"]
# Convert target country code to target country
if trgt in EIBI_Countries:
trgt = EIBI_Countries[trgt]
# Append a new entry to the result
result.append({
"freq" : int(float(m.group(1)) * 1000),
"freq" : freq,
"mode" : mode,
"time1" : int(m.group(2)),
"time2" : int(m.group(3)),
"days" : self.convertDays(m.group(4)),
"days" : self.convertDays(days),
"itu" : m.group(5),
"name" : m.group(6),
"lang" : m.group(7),
"tgt" : m.group(8),
"lang" : lang,
"tgt" : trgt,
"src" : m.group(9),
"p" : m.group(10),
"start" : m.group(11),
"stop" : m.group(12),
"pers" : int(m.group(10)),
"date1" : self.convertDate(m.group(11)),
"date2" : self.convertDate(m.group(12)),
})
except Exception as e:
logger.debug("scrape() exception: {0}".format(e))
logger.debug("loadFromWeb() exception: {0}".format(e))
# Done
return result
#
# Normal days of the week
#
EIBI_Days = {
"Mo" : 1,
"Tu" : 2,
"We" : 3,
"Th" : 4,
"Fr" : 5,
"Sa" : 6,
"Su" : 7,
}
#
# Special Codes for the days field
#
EIBI_SpecialDays = {
"" : "1234567", # Empty field means whole week
"LSB" : "1234567", # Upper side band transmission
"USB" : "1234567", # Upper side band transmission
"alt" : "xxxxxxx", # Alternative frequency, usually not in use
"irr" : "xxxxxxx", # Irregular operation
"Haj" : "xxxxxxx", # Special Haj broadcast
"Ram" : "xxxxxxx", # Special Ramadan schedule
"tent" : "xxxxxxx", # Tentatively, check and report your observations
"test" : "xxxxxxx", # Test operation, may cease at any time
"harm" : ".......", # Harmonic signal (multiples of fundamental frequency)
"imod" : ".......", # Intermodulation signal
}
#
# Country Codes
#
EIBI_Countries = {
# Regions
"Af" : "Africa",
"Am" : "Americas",
"As" : "Asia",
"C..": "Central ..",
"CAf": "Central Africa",
"CAm": "Central America",
"CAs": "Central Asia",
"CEu": "Central Europe",
"Car": "Caribbean, Gulf of Mexico, Florida Waters",
"Cau": "Caucasus",
"CIS": "Commonwealth of Independent States (former Soviet Union)",
"CNA": "Central North America",
"E..": "East ..",
"EAf": "Eastern Africa",
"EAs": "Eastern Asia",
"EEu": "Eastern Europe",
"ENA": "Eastern North America",
"ENE": "East-Northeast",
"ESE": "East-Southeast",
"Eu" : "Europe, incl. North Africa / Middle East",
"FE" : "Far East",
"Glo": "World",
"In" : "Indian Subcontinent",
"LAm": "Latin America",
"ME" : "Middle East",
"N..": "North ..",
"NAf": "North Africa",
"NAm": "North America",
"NAs": "North Asia",
"NEu": "North Europe",
"NAO": "North Atlantic Ocean",
"NE" : "Northeast",
"NNE": "North-Northeast",
"NNW": "North-Northwest",
"NW" : "Northwest",
"Oc" : "Oceania (Australia, New Zealand, Pacific Ocean)",
"S..": "South ..",
"SAf": "South Africa",
"SAm": "South America",
"SAs": "South Asia",
"SEu": "South Europe",
"SAO": "South Atlantic Ocean",
"SE" : "Southeast",
"SEA": "South East Asia",
"SEE": "South East Europe",
"Sib": "Siberia",
"SSE": "South-Southeast",
"SSW": "South-Southwest",
"SW" : "Southwest",
"Tib": "Tibet",
"W..": "West ..",
"WAf": "Western Africa",
"WAs": "Western Asia",
"WEu": "Western Europe",
"WIO": "Western Indian Ocean",
"WNA": "Western North America",
"WNW": "West-Northwest",
"WSW": "West-Southwest",
# ITU codes start here
"ABW": "Aruba",
"AFG": "Afghanistan",
"AFS": "South Africa",
@ -267,17 +439,17 @@ EIBI_Countries = {
"BTN": "Bhutan",
"BUL": "Bulgaria",
"BVT": "Bouvet",
"CAB": "Cabinda *",
"CAB": "Cabinda",
"CAF": "Central African Republic",
"CAN": "Canada",
"CBG": "Cambodia",
"CEU": "Ceuta *",
"CEU": "Ceuta",
"CG7": "Guantanamo Bay",
"CHL": "Chile",
"CHN": "China (People's Republic)",
"CHR": "Christmas Island (Indian Ocean)",
"CHN": "People's Republic of China",
"CHR": "Christmas Island in Indian Ocean",
"CKH": "Cook Island",
"CLA": "Clandestine stations *",
"CLA": "Clandestine stations",
"CLM": "Colombia",
"CLN": "Sri Lanka",
"CME": "Cameroon",
@ -307,7 +479,7 @@ EIBI_Countries = {
"ERI": "Eritrea",
"EST": "Estonia",
"ETH": "Ethiopia",
"EUR": "Iles Europe & Bassas da India *",
"EUR": "Iles Europe & Bassas da India",
"F": "France",
"FIN": "Finland",
"FJI": "Fiji",
@ -323,7 +495,7 @@ EIBI_Countries = {
"GMB": "Gambia",
"GNB": "Guinea-Bissau",
"GNE": "Equatorial Guinea",
"GPG": "Galapagos *",
"GPG": "Galapagos",
"GRC": "Greece",
"GRD": "Grenada",
"GRL": "Greenland",
@ -332,7 +504,7 @@ EIBI_Countries = {
"GUI": "Guinea",
"GUM": "Guam / Guahan",
"GUY": "Guyana",
"HKG": "Hong Kong, part of China",
"HKG": "Hong Kong",
"HMD": "Heard & McDonald Islands",
"HND": "Honduras",
"HNG": "Hungary",
@ -351,16 +523,16 @@ EIBI_Countries = {
"ISL": "Iceland",
"ISR": "Israel",
"IW": "International Waters",
"IWA": "Ogasawara (Bonin, Iwo Jima) *",
"IWA": "Ogasawara (Bonin, Iwo Jima)",
"J": "Japan",
"JAR": "Jarvis Island",
"JDN": "Juan de Nova *",
"JDN": "Juan de Nova",
"JMC": "Jamaica",
"JMY": "Jan Mayen *",
"JMY": "Jan Mayen",
"JON": "Johnston Island",
"JOR": "Jordan",
"JUF": "Juan Fernandez Island *",
"KAL": "Kaliningrad *",
"JUF": "Juan Fernandez Island",
"KAL": "Kaliningrad",
"KAZ": "Kazakstan / Kazakhstan",
"KEN": "Kenya",
"KER": "Kerguelen",
@ -389,7 +561,7 @@ EIBI_Countries = {
"MDG": "Madagascar",
"MDR": "Madeira",
"MDW": "Midway Islands",
"MEL": "Melilla *",
"MEL": "Melilla",
"MEX": "Mexico",
"MHL": "Marshall Islands",
"MKD": "Macedonia (F.Y.R.)",
@ -434,7 +606,7 @@ EIBI_Countries = {
"POR": "Portugal",
"PRG": "Paraguay",
"PRU": "Peru",
"PRV": "Okino-Tori-Shima (Parece Vela) *",
"PRV": "Okino-Tori-Shima (Parece Vela)",
"PSE": "Palestine",
"PTC": "Pitcairn",
"PTR": "Puerto Rico",
@ -445,11 +617,11 @@ EIBI_Countries = {
"RRW": "Rwanda",
"RUS": "Russian Federation",
"S": "Sweden",
"SAP": "San Andres & Providencia *",
"SAP": "San Andres & Providencia",
"SDN": "Sudan",
"SEN": "Senegal",
"SEY": "Seychelles",
"SGA": "South Georgia Islands *",
"SGA": "South Georgia Islands",
"SHN": "Saint Helena",
"SLM": "Solomon Islands",
"SLV": "El Salvador",
@ -457,17 +629,17 @@ EIBI_Countries = {
"SMO": "Samoa",
"SMR": "San Marino",
"SNG": "Singapore",
"SOK": "South Orkney Islands *",
"SOK": "South Orkney Islands",
"SOM": "Somalia",
"SPM": "Saint Pierre et Miquelon",
"SRB": "Serbia",
"SRL": "Sierra Leone",
"SSD": "South Sudan",
"SSI": "South Sandwich Islands *",
"SSI": "South Sandwich Islands",
"STP": "Sao Tome & Principe",
"SUI": "Switzerland",
"SUR": "Suriname",
"SVB": "Svalbard *",
"SVB": "Svalbard",
"SVK": "Slovakia",
"SVN": "Slovenia",
"SWZ": "Swaziland",
@ -487,12 +659,12 @@ EIBI_Countries = {
"TUN": "Tunisia",
"TUR": "Turkey",
"TUV": "Tuvalu",
"TWN": "Taiwan *",
"TWN": "Taiwan",
"TZA": "Tanzania",
"UAE": "United Arab Emirates",
"UGA": "Uganda",
"UKR": "Ukraine",
"UN": "United Nations *",
"UN": "United Nations",
"URG": "Uruguay",
"USA": "United States of America",
"UZB": "Uzbekistan",
@ -659,6 +831,8 @@ EIBI_Languages = {
"DY": { "name": "Dyula/Jula: Burkina Faso (1m), Ivory Coast (1.5m), Mali (50,000)", "code": "dyu" },
"DZ": { "name": "Dzongkha: Bhutan (0.2m)", "code": "dzo" },
"E": { "name": "English: UK (60m), USA (225m), India (200m), others", "code": "eng" },
"E,F": { "name": "English, French" },
"E,S": { "name": "English, Spanish" },
"EC": { "name": "Eastern Cham: Vietnam (70,000)", "code": "cjm" },
"EGY": { "name": "Egyptian Arabic: Egypt (52m)", "code": "arz" },
"EO": { "name": "Esperanto: Constructed language (2m)", "code": "epo" },

2
owrx/map.py
View File

@ -87,7 +87,7 @@ class Map(object):
# if location is permanent, shift its timestamp into the future
if permanent:
ts = ts + timedelta(weeks=1000)
ts = ts + timedelta(weeks=500)
with self.positionsLock:
# ignore indirect reports if ignoreIndirect set

184
owrx/markers.py
View File

@ -2,7 +2,8 @@ from owrx.config.core import CoreConfig
from owrx.map import Map, Location
from owrx.aprs import getSymbolData
from json import JSONEncoder
from owrx.eibi import EIBI_Locations
from owrx.eibi import EIBI_Locations, EIBI
from datetime import datetime
import urllib
import threading
@ -65,7 +66,9 @@ class Markers(object):
def __init__(self):
self.refreshPeriod = 60*60*24
self.event = threading.Event()
self.markers = {}
self.fmarkers = {}
self.wmarkers = {}
self.smarkers = {}
self.thread = None
# Known database files
self.fileList = [
@ -81,9 +84,6 @@ class Markers(object):
except Exception:
pass
def toJSON(self):
return self.markers
# Start the main thread
def startThread(self):
if self.thread is None:
@ -102,51 +102,116 @@ class Markers(object):
logger.debug("Starting marker database thread...")
# No markers yet
self.markers = {}
self.markers = {}
self.rxmarkers = {}
self.txmarkers = {}
# Load markers from local files
# Load miscellaneous markers from local files
for file in self.fileList:
if os.path.isfile(file):
logger.debug("Loading markers from '{0}'...".format(file))
self.markers.update(self.loadMarkers(file))
# Load markers from the EIBI database
#logger.debug("Loading EIBI transmitter locations...")
#self.markers.update(self.loadEIBI())
# This file contains cached database
# This file contains cached receivers database
file = self._getCachedMarkersFile()
ts = os.path.getmtime(file) if os.path.isfile(file) else 0
# Try loading cached database from file first, unless stale
if time.time() - ts < self.refreshPeriod:
logger.debug("Loading cached markers from '{0}'...".format(file))
self.markers.update(self.loadMarkers(file))
else:
# Add scraped data to the database
self.markers.update(self.updateCache())
# If cached receivers database stale, update it
if time.time() - ts >= self.refreshPeriod:
self.rxmarkers = self.updateCache()
ts = os.path.getmtime(file) if os.path.isfile(file) else 0
# If receivers database update did not run or failed, use cache
if not self.rxmarkers:
self.rxmarkers = self.loadMarkers(file)
# Load current schedule from the EIBI database
self.txmarkers = self.loadCurrentTransmitters()
# Update map with markers
logger.debug("Updating map...")
self.updateMap(self.markers)
self.updateMap(self.rxmarkers)
self.updateMap(self.txmarkers)
#
# Main Loop
#
while not self.event.is_set():
# Update map with markers
logger.debug("Updating map...")
self.updateMap()
# Sleep until it is time to update schedule
self.event.wait(self.refreshPeriod)
# If not terminated yet...
if not self.event.is_set():
# Scrape data, updating cache
logger.debug("Refreshing marker database...")
self.markers.update(self.updateCache())
# Wait for the head of the next hour
self.event.wait((60 - datetime.utcnow().minute) * 60)
if self.event.is_set():
break
# Load new transmitters schedule from the EIBI
logger.debug("Refreshing transmitters schedule..")
tx = self.loadCurrentTransmitters()
# Check if we need to exit
if self.event.is_set():
break
# Remove station markers that have no transmissions
map = Map.getSharedInstance()
notx = [x for x in self.txmarkers.keys() if x not in tx]
for key in notx:
map.removeLocation(key)
del self.txmarkers[key]
# Update station markers that have transmissions
for key in tx.keys():
r = tx[key]
map.updateLocation(r.getId(), r, r.getMode(), permanent=True)
self.txmarkers[key] = r
# Done with the schedule
notx = None
tx = None
# Check if we need to exit
if self.event.is_set():
break
# Update cached receivers data
if time.time() - ts >= self.refreshPeriod:
logger.debug("Refreshing receivers database...")
rx = self.updateCache()
ts = os.path.getmtime(file)
if rx:
# Remove receiver markers that no longer exist
norx = [x for x in self.rxmarkers.keys() if x not in rx]
for key in norx:
map.removeLocation(key)
del self.rxmarkers[key]
# Update receiver markers that are online
for key in rx.keys():
r = rx[key]
map.updateLocation(r.getId(), r, r.getMode(), permanent=True)
self.rxmarkers[key] = r
# Done updating receivers
norx = None
rx = None
# Done with the thread
logger.debug("Stopped marker database thread.")
self.thread = None
# Save markers to a given file
def saveMarkers(self, file: str, markers):
logger.debug("Saving {0} markers to '{1}'...".format(len(markers), file))
try:
with open(file, "w") as f:
json.dump(markers, f, cls=MyJSONEncoder, indent=2)
f.close()
except Exception as e:
logger.debug("saveMarkers() exception: {0}".format(e))
# Load markers from a given file
def loadMarkers(self, fileName: str):
def loadMarkers(self, file: str):
logger.debug("Loading markers from '{0}'...".format(file))
# Load markers list from JSON file
try:
with open(fileName, "r") as f:
with open(file, "r") as f:
db = json.load(f)
f.close()
except Exception as e:
@ -160,11 +225,12 @@ class Markers(object):
result[key] = MarkerLocation(attrs)
# Done
logger.debug("Loaded {0} markers from '{1}'.".format(len(result), file))
return result
# Update markers on the map
def updateMap(self):
for r in self.markers.values():
# Update given markers on the map
def updateMap(self, markers):
for r in markers.values():
Map.getSharedInstance().updateLocation(r.getId(), r, r.getMode(), permanent=True)
# Scrape online databases, updating cache file
@ -178,14 +244,11 @@ class Markers(object):
cache.update(self.scrapeWebSDR())
logger.debug("Scraping OpenWebRX website...")
cache.update(self.scrapeOWRX())
# Save parsed data into a file
logger.debug("Saving {0} markers to '{1}'...".format(len(cache), file))
try:
with open(file, "w") as f:
json.dump(cache, f, cls=MyJSONEncoder, indent=2)
f.close()
except Exception as e:
logger.debug("updateCache() exception: {0}".format(e))
# Save parsed data into a file, if there is anything to save
if cache:
self.saveMarkers(file, cache)
# Done
return cache
@ -193,23 +256,50 @@ class Markers(object):
# Following functions scrape data from websites and internal databases
#
def loadEIBI(self):
def loadCurrentTransmitters(self):
#url = "https://www.short-wave.info/index.php?txsite="
url = "https://www.google.com/search?q="
result = {}
# Refresh / load EIBI database, as needed
EIBI.getSharedInstance().refresh()
# Load transmitter sites from EIBI database
for entry in EIBI_Locations:
for entry in EIBI.getSharedInstance().currentTransmitters().values():
# Extract target regions and languages, removing duplicates
schedule = entry["schedule"]
langs = {}
targets = {}
comment = ""
langstr = ""
for row in schedule:
lang = row["lang"]
target = row["tgt"]
if target not in targets:
targets[target] = True
comment += (", " if comment else " to ") + target
if lang not in langs:
langs[lang] = True
langstr += (", " if langstr else "") + re.sub(r"(:|\s*\().*$", "", lang)
# Compose comment
comment = "Transmitting" + comment if comment else "Transmitter"
comment = comment + " (" + langstr + ")" if langstr else comment
rl = MarkerLocation({
"type" : "feature",
"mode" : "Stations",
"comment" : "Transmitter",
"comment" : comment,
"id" : entry["name"],
"lat" : entry["lat"],
"lon" : entry["lon"],
"url" : url + urllib.parse.quote_plus(entry["name"])
"url" : url + urllib.parse.quote_plus(entry["name"]),
"schedule": schedule
})
result[rl.getId()] = rl
# Done
logger.debug("Loaded {0} transmitters from EIBI.".format(len(result)))
return result
def scrapeOWRX(self, url: str = "https://www.receiverbook.de/map"):

2
owrx/version.py
View File

@ -1,5 +1,5 @@
from distutils.version import LooseVersion
_versionstring = "1.2.23"
_versionstring = "1.2.24"
looseversion = LooseVersion(_versionstring)
openwebrx_version = "v{0}".format(looseversion)