ua1zbe
/
openwebrxplus
mirror of https://github.com/luarvique/openwebrx
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Added Storage API for opening files that avoids name collisions.

This commit is contained in:
Marat Fayzullin 2024-01-08 22:36:47 -05:00
parent dec17d0228
commit b13447a134
6 changed files with 89 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
owrx/controllers/file.py
View File

@ -6,14 +6,14 @@ import re
class FileController(AssetsController):
def getFilePath(self, file):
return Storage().getFilePath(file)
return Storage.getFilePath(file)
class FilesController(WebpageController):
def template_variables(self):
isimg = re.compile(r'.*\.(png|bmp|gif|jpg)$')
issnd = re.compile(r'.*\.(mp3|wav)$')
files = Storage().getStoredFiles()
files = Storage.getSharedInstance().getStoredFiles()
rows = ""
for i in range(len(files)):

21
owrx/fax.py
View File

@ -32,19 +32,19 @@ class FaxParser(ThreadModule):
def closeFile(self):
if self.file is not None:
try:
logger.debug("Closing bitmap file '%s'." % self.fileName)
logger.debug("Closing bitmap file '%s'." % self.file.name)
self.file.close()
self.file = None
if self.height==0 or self.line<self.height:
logger.debug("Deleting short bitmap file '%s'." % self.fileName)
os.unlink(self.fileName)
logger.debug("Deleting short bitmap file '%s'." % self.file.name)
os.unlink(self.file.name)
else:
# Convert file from BMP to PNG
logger.debug("Converting '%s' to PNG..." % self.fileName)
Storage().convertImage(self.fileName)
logger.debug("Converting '%s' to PNG..." % self.file.name)
Storage.convertImage(self.file.name)
# Delete excessive files from storage
logger.debug("Performing storage cleanup...")
Storage().cleanStoredFiles()
Storage.getSharedInstance().cleanStoredFiles()
except Exception as e:
logger.debug("Exception closing file: %s" % str(e))
@ -53,9 +53,8 @@ class FaxParser(ThreadModule):
def newFile(self, fileName):
self.closeFile()
try:
self.fileName = Storage().getFilePath(fileName + ".bmp")
logger.debug("Opening bitmap file '%s'..." % self.fileName)
self.file = open(self.fileName, "wb")
logger.debug("Opening bitmap file '%s'..." % fileName)
self.file = Storage.getSharedInstance().newFile(fileName)
except Exception as e:
logger.debug("Exception opening file: %s" % str(e))
@ -207,7 +206,7 @@ class FaxParser(ThreadModule):
# Find mode name and time
modeName = "IOC-%d %dLPM" % (self.ioc, self.lpm)
timeStamp = datetime.utcnow().strftime("%H:%M:%S")
fileName = Storage().makeFileName("FAX-{0}", self.frequency)
fileName = Storage.makeFileName("FAX-{0}", self.frequency)
logger.debug("%s receiving %dx%d %s frame as '%s'." % (
self.myName(), self.width, self.height,
modeName, fileName
@ -215,7 +214,7 @@ class FaxParser(ThreadModule):
# If running as a service...
if self.service:
# Create a new image file and write BMP header
self.newFile(fileName)
self.newFile(fileName + ".bmp")
self.writeFile(self.data[0:headerSize])
# Empty result
out = {}

2
owrx/http.py
View File

@ -101,7 +101,7 @@ class Router(object):
StaticRoute("/policy", PolicyController),
StaticRoute("/features", FeatureController),
StaticRoute("/files", FilesController),
RegexRoute("^/files/(%s)$" % Storage().getNamePattern(), FileController),
RegexRoute("^/files/(%s)$" % Storage.getNamePattern(), FileController),
StaticRoute("/api/features", ApiController),
StaticRoute("/metrics", MetricsController, options={"action": "prometheusAction"}),
StaticRoute("/metrics.json", MetricsController),

21
owrx/sstv.py
View File

@ -75,19 +75,19 @@ class SstvParser(ThreadModule):
def closeFile(self):
if self.file is not None:
try:
logger.debug("Closing bitmap file '%s'." % self.fileName)
logger.debug("Closing bitmap file '%s'." % self.file.name)
self.file.close()
self.file = None
if self.height==0 or self.line<self.height:
logger.debug("Deleting short bitmap file '%s'." % self.fileName)
os.unlink(self.fileName)
logger.debug("Deleting short bitmap file '%s'." % self.file.name)
os.unlink(self.file.name)
else:
# Convert file from BMP to PNG
logger.debug("Converting '%s' to PNG..." % self.fileName)
Storage().convertImage(self.fileName)
logger.debug("Converting '%s' to PNG..." % self.file.name)
Storage.convertImage(self.file.name)
# Delete excessive files from storage
logger.debug("Performing storage cleanup...")
Storage().cleanStoredFiles()
Storage.getSharedInstance().cleanStoredFiles()
except Exception as exptn:
logger.debug("Exception closing file: %s" % str(exptn))
@ -96,9 +96,8 @@ class SstvParser(ThreadModule):
def newFile(self, fileName):
self.closeFile()
try:
self.fileName = Storage().getFilePath(fileName + ".bmp")
logger.debug("Opening bitmap file '%s'..." % self.fileName)
self.file = open(self.fileName, "wb")
logger.debug("Opening bitmap file '%s'..." % fileName)
self.file = Storage.getSharedInstance().newFile(fileName)
except Exception as exptn:
logger.debug("Exception opening file: %s" % str(exptn))
@ -231,7 +230,7 @@ class SstvParser(ThreadModule):
# Find mode name and time
modeName = modeNames.get(self.mode) if self.mode in modeNames else "Unknown Mode %d" % self.mode
timeStamp = datetime.utcnow().strftime("%H:%M:%S")
fileName = Storage().makeFileName("SSTV-{0}", self.frequency)
fileName = Storage.makeFileName("SSTV-{0}", self.frequency)
logger.debug("%s receiving %dx%d %s frame as '%s'." % (
self.myName(), self.width, self.height,
modeName, fileName
@ -239,7 +238,7 @@ class SstvParser(ThreadModule):
# If running as a service...
if self.service:
# Create a new image file and write BMP header
self.newFile(fileName)
self.newFile(fileName + ".bmp")
self.writeFile(self.data[0:54])
# Empty result
out = {}

84
owrx/storage.py
View File

@ -2,6 +2,7 @@ from owrx.config.core import CoreConfig
from owrx.config import Config
from datetime import datetime
import threading
import subprocess
import os
import re
@ -11,30 +12,44 @@ import logging
logger = logging.getLogger(__name__)
class Storage(object):
sharedInstance = None
creationLock = threading.Lock()
filePattern = r'[A-Z0-9]+-[0-9]+-[0-9]+(-[0-9]+)?(-[0-9]+)?\.(bmp|png|txt|mp3)'
# Get shared instance of Storage class
@staticmethod
def getSharedInstance():
with Storage.creationLock:
if Storage.sharedInstance is None:
Storage.sharedInstance = Storage()
return Storage.sharedInstance
# Construct an instance of Storage class
def __init__(self):
self.filePattern = r'[A-Z0-9]+-[0-9]+-[0-9]+(-[0-9]+)?\.(bmp|png|txt|mp3)'
self.lock = threading.Lock()
# Get file name pattern
def getNamePattern(self):
return self.filePattern
# Create stored file by name, modifying the name if the file exists
def newFile(self, fileName: str, buffering: int = -1):
filePath = self.getFilePath(fileName)
# Create stored file name by inserting current UTC date
# and time into the pattern spot designated with "{0}"
def makeFileName(self, pattern: str = '{0}', frequency: int = 0):
d = datetime.utcnow().strftime('%y%m%d-%H%M%S')
f = ('-%d' % (frequency // 1000)) if frequency>0 else ''
return pattern.format(d + f)
with self.lock:
if not os.path.exists(filePath):
return open(filePath, "wb", buffering = buffering)
elif filePath.contains("."):
filePathX = "-{0}.".join(filePath.rsplit(".", 1))
for i in range(99):
filePath1 = filePathX.format(i)
if not os.path.exists(filePath1):
return open(filePath1, "wb", buffering = buffering)
# Get complete path to a stored file from its filename by
# adding folder name
def getFilePath(self, filename: str):
return os.path.join(CoreConfig().get_temporary_directory(), filename)
raise FileExistsError("File '{0}' already exists.".format(filePath))
# Get list of stored files, sorted in reverse alphabetic order
# (so that newer files appear first)
def getStoredFiles(self):
dir = CoreConfig().get_temporary_directory()
files = [os.path.join(dir, f) for f in os.listdir(dir) if re.match(self.filePattern, f)]
with self.lock:
files = [os.path.join(dir, f) for f in os.listdir(dir) if re.match(self.filePattern, f)]
files.sort(key=lambda x: os.path.getmtime(x), reverse=True)
return [os.path.basename(f) for f in files]
@ -43,16 +58,39 @@ class Storage(object):
pm = Config.get()
keep = pm["keep_files"]
dir = CoreConfig().get_temporary_directory()
files = [os.path.join(dir, f) for f in os.listdir(dir) if re.match(self.filePattern, f)]
files.sort(key=lambda x: os.path.getmtime(x), reverse=True)
for f in files[keep:]:
logger.debug("Deleting stored file '%s'." % os.path.basename(f))
try:
os.unlink(f)
except Exception as e:
logger.debug("cleanStoredFiles(): " + str(e))
with self.lock:
files = [os.path.join(dir, f) for f in os.listdir(dir) if re.match(self.filePattern, f)]
files.sort(key=lambda x: os.path.getmtime(x), reverse=True)
for f in files[keep:]:
logger.debug("Deleting stored file '%s'." % os.path.basename(f))
try:
os.unlink(f)
except Exception as e:
logger.debug("cleanStoredFiles(): " + str(e))
# Get file name pattern
@staticmethod
def getNamePattern():
return Storage.filePattern
# Get complete path to a stored file from its filename by
# adding folder name
@staticmethod
def getFilePath(filename: str):
return os.path.join(CoreConfig().get_temporary_directory(), filename)
# Create stored file name by inserting current UTC date
# and time into the pattern spot designated with "{0}"
@staticmethod
def makeFileName(pattern: str = '{0}', frequency: int = 0):
d = datetime.utcnow().strftime('%y%m%d-%H%M%S')
f = ('-%d' % (frequency // 1000)) if frequency>0 else ''
return pattern.format(d + f)
# Convert given file from BMP to PNG format using ImageMagick
@staticmethod
def convertImage(self, inFile: str):
# Adds storage path
if not inFile.startswith('/'):

11
owrx/toolbox.py
View File

@ -32,12 +32,12 @@ class TextParser(LineBasedModule):
def closeFile(self):
if self.file is not None:
try:
logger.debug("Closing log file '%s'." % self.fileName)
logger.debug("Closing log file '%s'." % self.file.name)
self.file.close()
self.file = None
# Delete excessive files from storage
logger.debug("Performing storage cleanup...")
Storage().cleanStoredFiles()
Storage.getSharedInstance().cleanStoredFiles()
except Exception as exptn:
logger.debug("Exception closing file: %s" % str(exptn))
@ -46,9 +46,8 @@ class TextParser(LineBasedModule):
def newFile(self, fileName):
self.closeFile()
try:
self.fileName = Storage().getFilePath(fileName + ".txt")
logger.debug("Opening log file '%s'..." % self.fileName)
self.file = open(self.fileName, "wb", buffering = 0)
logger.debug("Opening log file '%s'..." % fileName)
self.file = Storage.getSharedInstance().newFile(fileName, buffering = 0)
self.cntLines = 0
except Exception as exptn:
@ -58,7 +57,7 @@ class TextParser(LineBasedModule):
def writeFile(self, data):
# If no file open, create and open a new file
if self.file is None and self.filePfx is not None:
self.newFile(Storage().makeFileName(self.filePfx+"-{0}", self.frequency))
self.newFile(Storage.makeFileName(self.filePfx+"-{0}", self.frequency) + ".txt")
# If file open now...
if self.file is not None:
# Write new line into the file